{"meta":{"date":"2026-08-25","timezone":"UTC","generated_at":"2026-08-25T21:12:29Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":133,"failed_runs":120,"runs_with_occurrences":119,"runs_with_failure_pattern_matches":80,"runs_with_unmatched_occurrences":39,"runs_without_occurrence_rows":14,"failed_runs_without_occurrence_rows":1},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092326377335296000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"","pr_sha":"427763772c2cd234e542a70b64cad19c14d6c447","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T19:00:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"failed_without_raw_rows"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6681,"pr_state":"open","pr_sha":"6a88f9ab6609f6b5ea3241e0669710d8aba0079a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:57:53Z"},"failed_at":["alert","e2e"],"failed_test_count":54,"occurrences":[{"row_id":"00af413cbb9bc1f51cb42a4d9df81ff4f4e4f5d99cc39b7d2895087010e921a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"e5bdbec92086140fc310730009e23b06059442f01f5776e5f1192f7ae25b3f0f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-v6268v with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016f6480\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v6268v: failed waiting for cluster=\"np-version-upgrade-cluster-v6268v\" in resourcegroup=\"rg-np-version-upgrade-v6268v-nt2n8g9nqgs6\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccf1b6dd-e00f-4656-9dc5-aa80d3952678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccf1b6dd-e00f-4656-9dc5-aa80d3952678\",\n \"name\": \"ccf1b6dd-e00f-4656-9dc5-aa80d3952678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.590603383Z\",\n \"endTime\": \"2026-08-25T20:07:57.191296022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-v6268v with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016f6480\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v6268v: failed waiting for cluster=\"np-version-upgrade-cluster-v6268v\" in resourcegroup=\"rg-np-version-upgrade-v6268v-nt2n8g9nqgs6\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccf1b6dd-e00f-4656-9dc5-aa80d3952678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccf1b6dd-e00f-4656-9dc5-aa80d3952678\",\n \"name\": \"ccf1b6dd-e00f-4656-9dc5-aa80d3952678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.590603383Z\",\n \"endTime\": \"2026-08-25T20:07:57.191296022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"14da63146187e36cd41025259a280a252c5f6639bc1534c06c3a796b40f13c98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"4ba0a65bf49c66d944c03681270514049220b21efd9126ab47abc5555d712dc1","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013503c0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-k2wkzd68x6rw\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"name\": \"d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.9471217Z\",\n \"endTime\": \"2026-08-25T20:07:47.037810744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013503c0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-k2wkzd68x6rw\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"name\": \"d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.9471217Z\",\n \"endTime\": \"2026-08-25T20:07:47.037810744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"164631cde5924fa9af02061dc8df5ac0b8bfee9006e81248778dbd63067d12cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"50e102a0b8a908cb91efb330c142288a7922ca39527756e228d95800edd32777","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001058120\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-mc5gjg8ldxj4\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b27aa63-ee26-4cb0-b825-8ba527fa8e1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b27aa63-ee26-4cb0-b825-8ba527fa8e1a\",\n \"name\": \"4b27aa63-ee26-4cb0-b825-8ba527fa8e1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.213415547Z\",\n \"endTime\": \"2026-08-25T20:07:57.181623203Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001058120\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-mc5gjg8ldxj4\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b27aa63-ee26-4cb0-b825-8ba527fa8e1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b27aa63-ee26-4cb0-b825-8ba527fa8e1a\",\n \"name\": \"4b27aa63-ee26-4cb0-b825-8ba527fa8e1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.213415547Z\",\n \"endTime\": \"2026-08-25T20:07:57.181623203Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16d2f4490e726d4310a504cecf6a6854bc8c79447bb474fedcd5a5154c39b212","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"a6c110259b6db6815f32629ea3be7e1b59c5bf8c1cc0025a60f627f7d9a52eb7","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000582680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-x5nktxhv4rqc\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb395680-8b7d-4a87-829d-dc9ed18d3d16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb395680-8b7d-4a87-829d-dc9ed18d3d16\",\n \"name\": \"bb395680-8b7d-4a87-829d-dc9ed18d3d16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:25.654631037Z\",\n \"endTime\": \"2026-08-25T20:07:27.051715102Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000582680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-x5nktxhv4rqc\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb395680-8b7d-4a87-829d-dc9ed18d3d16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb395680-8b7d-4a87-829d-dc9ed18d3d16\",\n \"name\": \"bb395680-8b7d-4a87-829d-dc9ed18d3d16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:25.654631037Z\",\n \"endTime\": \"2026-08-25T20:07:27.051715102Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"17973b779263b982000985bec14dedf2397757ae2eab29c03e1697fd023cc125","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"a2da6b6b096cccb301a6ba7a740e8c3e59f518fd173797735af9ce7fdb6f0758","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-dr999r\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00119c320\u003e: \n failed to create HCP cluster cluster-zstream-4-21-dr999r: failed waiting for cluster=\"cluster-zstream-4-21-dr999r\" in resourcegroup=\"rg-zstream-upgrade-4-21-dw27j4dxzb2w\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56417240-6514-4115-b8c2-510281e04e26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56417240-6514-4115-b8c2-510281e04e26\",\n \"name\": \"56417240-6514-4115-b8c2-510281e04e26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.027389455Z\",\n \"endTime\": \"2026-08-25T20:07:46.859936016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-dr999r\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00119c320\u003e: \n failed to create HCP cluster cluster-zstream-4-21-dr999r: failed waiting for cluster=\"cluster-zstream-4-21-dr999r\" in resourcegroup=\"rg-zstream-upgrade-4-21-dw27j4dxzb2w\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56417240-6514-4115-b8c2-510281e04e26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56417240-6514-4115-b8c2-510281e04e26\",\n \"name\": \"56417240-6514-4115-b8c2-510281e04e26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.027389455Z\",\n \"endTime\": \"2026-08-25T20:07:46.859936016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"17a5351c1bb4803b3c4b077a5a48b6867fab581bbc0c7cbf40a5647676a07cc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"d550d5cb6f50d0f9ff8f38ae93cfebf0a70e58547865e6dbf73e9cbe1c055656","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c7e280\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-xthbgmp5jqk5\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee494a16-dcba-42ec-9b37-be5f13c78acc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee494a16-dcba-42ec-9b37-be5f13c78acc\",\n \"name\": \"ee494a16-dcba-42ec-9b37-be5f13c78acc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.140574044Z\",\n \"endTime\": \"2026-08-25T20:07:46.990280541Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c7e280\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-xthbgmp5jqk5\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee494a16-dcba-42ec-9b37-be5f13c78acc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee494a16-dcba-42ec-9b37-be5f13c78acc\",\n \"name\": \"ee494a16-dcba-42ec-9b37-be5f13c78acc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.140574044Z\",\n \"endTime\": \"2026-08-25T20:07:46.990280541Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1e14ed2fb62714b75ccde9ba7e9029a73ae72367bd2791d41c0e7b343d7c45b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"071c52a55532188ca4d85bb7a66cc6fe5057e406feb7fa0eb18ba541618e910d","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28d80\u003e: \n failed to create HCP cluster np-dg-minor-zpsckm: failed waiting for cluster=\"np-dg-minor-zpsckm\" in resourcegroup=\"rg-np-dg-minor-zpsckm-7j842f42696c\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"name\": \"c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.123542143Z\",\n \"endTime\": \"2026-08-25T20:07:46.9158638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28d80\u003e: \n failed to create HCP cluster np-dg-minor-zpsckm: failed waiting for cluster=\"np-dg-minor-zpsckm\" in resourcegroup=\"rg-np-dg-minor-zpsckm-7j842f42696c\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"name\": \"c906de65-e3a2-4879-9236-7b176a8d1899\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.123542143Z\",\n \"endTime\": \"2026-08-25T20:07:46.9158638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"22303b098b40e0ea43073530ae198e696d71ffb1634e39ef3abfb692ff7c5033","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"ade895469bda3f6ff370f0428733134c672c1f332d14dbed05c869078e270076","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-mmxsk9\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016a4d20\u003e: \n failed to create HCP cluster cluster-zstream-4-22-mmxsk9: failed waiting for cluster=\"cluster-zstream-4-22-mmxsk9\" in resourcegroup=\"rg-zstream-upgrade-4-22-7sw6gg5nrfgp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e92c28a-59f3-43ef-8a86-1e504c2c3fb4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e92c28a-59f3-43ef-8a86-1e504c2c3fb4\",\n \"name\": \"6e92c28a-59f3-43ef-8a86-1e504c2c3fb4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.508111897Z\",\n \"endTime\": \"2026-08-25T20:07:47.051363953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-mmxsk9\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016a4d20\u003e: \n failed to create HCP cluster cluster-zstream-4-22-mmxsk9: failed waiting for cluster=\"cluster-zstream-4-22-mmxsk9\" in resourcegroup=\"rg-zstream-upgrade-4-22-7sw6gg5nrfgp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e92c28a-59f3-43ef-8a86-1e504c2c3fb4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e92c28a-59f3-43ef-8a86-1e504c2c3fb4\",\n \"name\": \"6e92c28a-59f3-43ef-8a86-1e504c2c3fb4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.508111897Z\",\n \"endTime\": \"2026-08-25T20:07:47.051363953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2f6a4d976667643babc850c2208353371c630b9d94f94569cc93e74a33de05a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"1b0503c4496d672dc2cfe86aa011acfb4fd95d2eec277298e9e02bf59cf65ab3","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015c0320\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-p27fqvr9ltxp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f93fe58-926a-482c-94ba-9753a5b36ccf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f93fe58-926a-482c-94ba-9753a5b36ccf\",\n \"name\": \"1f93fe58-926a-482c-94ba-9753a5b36ccf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.799125747Z\",\n \"endTime\": \"2026-08-25T20:07:47.040603791Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015c0320\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-p27fqvr9ltxp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f93fe58-926a-482c-94ba-9753a5b36ccf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f93fe58-926a-482c-94ba-9753a5b36ccf\",\n \"name\": \"1f93fe58-926a-482c-94ba-9753a5b36ccf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.799125747Z\",\n \"endTime\": \"2026-08-25T20:07:47.040603791Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"367188e1f255f9cde2475e485d7ac577fd3907d89a0ee828a0a8d98e0e8624b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"6fe96d1ea335b0c185768786273c3d8084095250845c72ff17619116c26601e4","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f68320\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6jx7b729v6jt\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f815a84f-9183-4d28-a908-84f86d986650\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f815a84f-9183-4d28-a908-84f86d986650\",\n \"name\": \"f815a84f-9183-4d28-a908-84f86d986650\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:57.296425588Z\",\n \"endTime\": \"2026-08-25T20:08:07.290237234Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f68320\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6jx7b729v6jt\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f815a84f-9183-4d28-a908-84f86d986650\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f815a84f-9183-4d28-a908-84f86d986650\",\n \"name\": \"f815a84f-9183-4d28-a908-84f86d986650\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:57.296425588Z\",\n \"endTime\": \"2026-08-25T20:08:07.290237234Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3766f7398099872811572539dbbdb0b73feab28fecb0810b7e1114054756e6f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"1babbd681ea180bcbac323e120929964548b008afb246353cbf1fad90e2045d0","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00148a4e0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vpqrhjtsh6sn\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e0a601a-840c-4990-aa7e-2925307335b5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e0a601a-840c-4990-aa7e-2925307335b5\",\n \"name\": \"2e0a601a-840c-4990-aa7e-2925307335b5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:18.947053934Z\",\n \"endTime\": \"2026-08-25T20:07:26.946534805Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00148a4e0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vpqrhjtsh6sn\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e0a601a-840c-4990-aa7e-2925307335b5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e0a601a-840c-4990-aa7e-2925307335b5\",\n \"name\": \"2e0a601a-840c-4990-aa7e-2925307335b5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:18.947053934Z\",\n \"endTime\": \"2026-08-25T20:07:26.946534805Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3acf50674b4d24830fa90af06b8c8c4819cdb7ff66016d298fdb4e4fef79ec3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"f70d11b5e9cc7db70c1bd07ccd81a8b490b5e1aa926781339d876d128c6ce56b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wznc5z\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f06180\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wznc5z: failed waiting for cluster=\"cluster-zstream-5-0-wznc5z\" in resourcegroup=\"rg-zstream-upgrade-5-0-ctmm2b9pmbd9\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f4a6ced-5900-41ba-af77-3ba2d089ff99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f4a6ced-5900-41ba-af77-3ba2d089ff99\",\n \"name\": \"3f4a6ced-5900-41ba-af77-3ba2d089ff99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.612594395Z\",\n \"endTime\": \"2026-08-25T20:07:57.106556779Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wznc5z\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f06180\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wznc5z: failed waiting for cluster=\"cluster-zstream-5-0-wznc5z\" in resourcegroup=\"rg-zstream-upgrade-5-0-ctmm2b9pmbd9\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f4a6ced-5900-41ba-af77-3ba2d089ff99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f4a6ced-5900-41ba-af77-3ba2d089ff99\",\n \"name\": \"3f4a6ced-5900-41ba-af77-3ba2d089ff99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.612594395Z\",\n \"endTime\": \"2026-08-25T20:07:57.106556779Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b88fd745a563c7b39d32f01c7168232765f4f0c51143e2bde3bbc129eda7422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"c73655be7d93d003560cdc90a2bab9e176e6bcf65d8c644387408b36f348155e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138e9a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9r5k552vk2zl\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46955ba2-9570-44ab-a359-565942f6b3ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46955ba2-9570-44ab-a359-565942f6b3ea\",\n \"name\": \"46955ba2-9570-44ab-a359-565942f6b3ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.300415435Z\",\n \"endTime\": \"2026-08-25T20:07:47.009449961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138e9a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9r5k552vk2zl\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46955ba2-9570-44ab-a359-565942f6b3ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46955ba2-9570-44ab-a359-565942f6b3ea\",\n \"name\": \"46955ba2-9570-44ab-a359-565942f6b3ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.300415435Z\",\n \"endTime\": \"2026-08-25T20:07:47.009449961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e92e3bfc041977e13b29839a666d13686bc7bcaccc5795aa30a3d7c6a8ba038","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"86d19c78d0739357bd3f161b74003da30c80a9d54ee056afba56d5e92cfbbc8a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-brvkkxwffwvv/cluster-candidate-4-21-rs5m87 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00165a120\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rs5m87: failed waiting for cluster=\"cluster-candidate-4-21-rs5m87\" in resourcegroup=\"rg-candidate-4-21-brvkkxwffwvv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78bcd8f5-f8e8-4195-9ada-f7d078d15259\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78bcd8f5-f8e8-4195-9ada-f7d078d15259\",\n \"name\": \"78bcd8f5-f8e8-4195-9ada-f7d078d15259\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.593126287Z\",\n \"endTime\": \"2026-08-25T20:07:57.185220723Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-brvkkxwffwvv/cluster-candidate-4-21-rs5m87 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00165a120\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rs5m87: failed waiting for cluster=\"cluster-candidate-4-21-rs5m87\" in resourcegroup=\"rg-candidate-4-21-brvkkxwffwvv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78bcd8f5-f8e8-4195-9ada-f7d078d15259\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78bcd8f5-f8e8-4195-9ada-f7d078d15259\",\n \"name\": \"78bcd8f5-f8e8-4195-9ada-f7d078d15259\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.593126287Z\",\n \"endTime\": \"2026-08-25T20:07:57.185220723Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"416e9d6c8feb66abef6a9f37ae791a535e0241fbc752d9ab75ad5ec9ad29469a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"2bae653a8fb17447152d646aa0c2b4963ab31a22269313b92437db057bf036df","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-v6rwcn\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010808c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-v6rwcn: failed waiting for cluster=\"cp-ystream-upgrade-4-22-v6rwcn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-lmsfsx99plpj\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c05d023b-5cbf-4f95-a639-811f615ee48f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c05d023b-5cbf-4f95-a639-811f615ee48f\",\n \"name\": \"c05d023b-5cbf-4f95-a639-811f615ee48f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:37.688782058Z\",\n \"endTime\": \"2026-08-25T20:07:46.986210564Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-v6rwcn\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010808c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-v6rwcn: failed waiting for cluster=\"cp-ystream-upgrade-4-22-v6rwcn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-lmsfsx99plpj\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c05d023b-5cbf-4f95-a639-811f615ee48f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c05d023b-5cbf-4f95-a639-811f615ee48f\",\n \"name\": \"c05d023b-5cbf-4f95-a639-811f615ee48f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:37.688782058Z\",\n \"endTime\": \"2026-08-25T20:07:46.986210564Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"49c0cbb85c7caadae85756d143d74eff0ca34ef1c1b277a1711ce230d07ea5c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"2c154aeee3d38aa0e65baa996b6067656fdcb76e0501965057b6de64078cad6d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5c2e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-796256lr558w\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b563dcd-8437-46c0-8193-3391fb7442fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b563dcd-8437-46c0-8193-3391fb7442fe\",\n \"name\": \"0b563dcd-8437-46c0-8193-3391fb7442fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:50:08.419900643Z\",\n \"endTime\": \"2026-08-25T20:09:09.088813522Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5c2e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-796256lr558w\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b563dcd-8437-46c0-8193-3391fb7442fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b563dcd-8437-46c0-8193-3391fb7442fe\",\n \"name\": \"0b563dcd-8437-46c0-8193-3391fb7442fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:50:08.419900643Z\",\n \"endTime\": \"2026-08-25T20:09:09.088813522Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d16e9b864b9f8d5238642e3b6b2de6ccb72d8570a96da6be64cbdcde24552b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"e7f3a8d6fb04405286572a2d49c4ffc07ae39f040f9b3c52e8290ec4d6f9c045","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000608d60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-x46mwn6rh7gp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/443cfc30-a943-4373-9d10-6810780445d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/443cfc30-a943-4373-9d10-6810780445d0\",\n \"name\": \"443cfc30-a943-4373-9d10-6810780445d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:39.894181304Z\",\n \"endTime\": \"2026-08-25T20:08:39.132471884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000608d60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-x46mwn6rh7gp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/443cfc30-a943-4373-9d10-6810780445d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/443cfc30-a943-4373-9d10-6810780445d0\",\n \"name\": \"443cfc30-a943-4373-9d10-6810780445d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:39.894181304Z\",\n \"endTime\": \"2026-08-25T20:08:39.132471884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4e6b4fd4e1dd12f8afc1efa3359f65f76f2ed9f99111dc294ce43c86944239f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"e92d23b1277ca5f619831904feaa802051ec36b3b4c1e38928537ad0c359a4e7","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001060540\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rckjxswkvsbp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d36c247-9bc7-41d9-8391-d00a410a8ab3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d36c247-9bc7-41d9-8391-d00a410a8ab3\",\n \"name\": \"9d36c247-9bc7-41d9-8391-d00a410a8ab3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.55718932Z\",\n \"endTime\": \"2026-08-25T20:07:47.034187037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001060540\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rckjxswkvsbp\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d36c247-9bc7-41d9-8391-d00a410a8ab3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d36c247-9bc7-41d9-8391-d00a410a8ab3\",\n \"name\": \"9d36c247-9bc7-41d9-8391-d00a410a8ab3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.55718932Z\",\n \"endTime\": \"2026-08-25T20:07:47.034187037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"527379749c730b58cd3563603ab5970b4c4a626724fd1087d43cd422b94fe118","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"e0b54c003aebec5d92b194a6f80ee906c128eb77169d7d2a7cba65e9aeeeba75","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002514840\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-k85zbvt86fnb\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0fd3271-7504-4bd7-8fa0-f1b585a65cab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0fd3271-7504-4bd7-8fa0-f1b585a65cab\",\n \"name\": \"e0fd3271-7504-4bd7-8fa0-f1b585a65cab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:50.424552123Z\",\n \"endTime\": \"2026-08-25T20:07:57.076670855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002514840\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-k85zbvt86fnb\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0fd3271-7504-4bd7-8fa0-f1b585a65cab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0fd3271-7504-4bd7-8fa0-f1b585a65cab\",\n \"name\": \"e0fd3271-7504-4bd7-8fa0-f1b585a65cab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:50.424552123Z\",\n \"endTime\": \"2026-08-25T20:07:57.076670855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"56831d66099daea3b0fea6b016d6ef3e82d97d234cd1b14101e61ad5240b88bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"37f401123d689a1b77a09fb3105cb5815a2f706e5e9877cf69d6ae68bf44f228","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a58440\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-k4hfm5tjnlzq\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd2e378-0451-47fb-91c8-3d5a113d1b92\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd2e378-0451-47fb-91c8-3d5a113d1b92\",\n \"name\": \"ccd2e378-0451-47fb-91c8-3d5a113d1b92\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.373976116Z\",\n \"endTime\": \"2026-08-25T20:07:47.047267279Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a58440\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-k4hfm5tjnlzq\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd2e378-0451-47fb-91c8-3d5a113d1b92\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd2e378-0451-47fb-91c8-3d5a113d1b92\",\n \"name\": \"ccd2e378-0451-47fb-91c8-3d5a113d1b92\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.373976116Z\",\n \"endTime\": \"2026-08-25T20:07:47.047267279Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"585d26bbec75bc2049ff9bdfb20348a89d7a234812359f1e92924598ab761849","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"150626fb402aa61e05e6c6f7e9bc8843969fdad7b45425051818fe06847d9754","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-q2758t47pkz2/cluster-candidate-4-20-jrg4sz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001806660\u003e: \n failed to create HCP cluster cluster-candidate-4-20-jrg4sz: failed waiting for cluster=\"cluster-candidate-4-20-jrg4sz\" in resourcegroup=\"rg-candidate-4-20-q2758t47pkz2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b0574d20-7b51-4455-80d3-3eed1d46d4d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b0574d20-7b51-4455-80d3-3eed1d46d4d4\",\n \"name\": \"b0574d20-7b51-4455-80d3-3eed1d46d4d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.030086536Z\",\n \"endTime\": \"2026-08-25T20:07:47.039102617Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-q2758t47pkz2/cluster-candidate-4-20-jrg4sz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001806660\u003e: \n failed to create HCP cluster cluster-candidate-4-20-jrg4sz: failed waiting for cluster=\"cluster-candidate-4-20-jrg4sz\" in resourcegroup=\"rg-candidate-4-20-q2758t47pkz2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b0574d20-7b51-4455-80d3-3eed1d46d4d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b0574d20-7b51-4455-80d3-3eed1d46d4d4\",\n \"name\": \"b0574d20-7b51-4455-80d3-3eed1d46d4d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.030086536Z\",\n \"endTime\": \"2026-08-25T20:07:47.039102617Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5ae2f96a8f3bdcf8ed444666c45f9c3d85081efed5971f6dfb049c9a7a3f836f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"227699a26b6d86c6ff9a362b2fd1395de244815866341bf1ceee8b2242c39b99","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-mxl8jv at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105c820\u003e: \n failed to create HCP cluster np-skip-minor-mxl8jv: failed waiting for cluster=\"np-skip-minor-mxl8jv\" in resourcegroup=\"rg-np-skip-minor-mxl8jv-b9n9487vtvr8\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8df00e8d-5e89-4c1a-942c-cec49ee723b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8df00e8d-5e89-4c1a-942c-cec49ee723b6\",\n \"name\": \"8df00e8d-5e89-4c1a-942c-cec49ee723b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:41.604813079Z\",\n \"endTime\": \"2026-08-25T20:07:46.915990537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-mxl8jv at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105c820\u003e: \n failed to create HCP cluster np-skip-minor-mxl8jv: failed waiting for cluster=\"np-skip-minor-mxl8jv\" in resourcegroup=\"rg-np-skip-minor-mxl8jv-b9n9487vtvr8\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8df00e8d-5e89-4c1a-942c-cec49ee723b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8df00e8d-5e89-4c1a-942c-cec49ee723b6\",\n \"name\": \"8df00e8d-5e89-4c1a-942c-cec49ee723b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:41.604813079Z\",\n \"endTime\": \"2026-08-25T20:07:46.915990537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c1a0fb71fb6d4de7f5a0bd22c89d4e778592820403b530592df284b5bdd4e8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"0bed7020ecd96a5cf05f223e463371381d77da2411a879375e1ceda63de28e4d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4dc85q\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001024220\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4dc85q: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4dc85q\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c6q8blh8xrtn\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba94d2d6-4fb5-425f-8b9e-2014d5d00330\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba94d2d6-4fb5-425f-8b9e-2014d5d00330\",\n \"name\": \"ba94d2d6-4fb5-425f-8b9e-2014d5d00330\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:36.642577785Z\",\n \"endTime\": \"2026-08-25T20:07:37.074791901Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4dc85q\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001024220\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4dc85q: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4dc85q\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c6q8blh8xrtn\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba94d2d6-4fb5-425f-8b9e-2014d5d00330\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba94d2d6-4fb5-425f-8b9e-2014d5d00330\",\n \"name\": \"ba94d2d6-4fb5-425f-8b9e-2014d5d00330\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:36.642577785Z\",\n \"endTime\": \"2026-08-25T20:07:37.074791901Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5f514eb696e16dc8ffebab0638642512452aa400510a189cbb1a4f6e6d9caaa1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"5c886e13528881fff7670bd0116e318e6fdf76957dfb81580b348ec9602154a5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ca460\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bdsb4flnlkm2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95f3031d-7183-4d81-8b51-115c49552af2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95f3031d-7183-4d81-8b51-115c49552af2\",\n \"name\": \"95f3031d-7183-4d81-8b51-115c49552af2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:48.199400026Z\",\n \"endTime\": \"2026-08-25T20:07:57.098721889Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ca460\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bdsb4flnlkm2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95f3031d-7183-4d81-8b51-115c49552af2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95f3031d-7183-4d81-8b51-115c49552af2\",\n \"name\": \"95f3031d-7183-4d81-8b51-115c49552af2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:48.199400026Z\",\n \"endTime\": \"2026-08-25T20:07:57.098721889Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6712b3ed4b976d0aabcd650afa6bd5db2318722c4256c6d927a2bc3dc4c9f54b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"137c0167738c3818a360a99bda994cbeeaf885b6a21dfa26c7f7e5381c79eb6a","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f94a80\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-jfpxlq9j587r\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa248783-1aa3-40f5-a3ef-143346f14307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa248783-1aa3-40f5-a3ef-143346f14307\",\n \"name\": \"fa248783-1aa3-40f5-a3ef-143346f14307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.285814221Z\",\n \"endTime\": \"2026-08-25T20:07:57.175172762Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f94a80\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-jfpxlq9j587r\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa248783-1aa3-40f5-a3ef-143346f14307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa248783-1aa3-40f5-a3ef-143346f14307\",\n \"name\": \"fa248783-1aa3-40f5-a3ef-143346f14307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.285814221Z\",\n \"endTime\": \"2026-08-25T20:07:57.175172762Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6796ef3f5e62d0856f83312efed489865eac536ba01e54fe2a921e2c6ff015b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"889088819d9dad08a48983198d0e17c9a48b8ac3d79ea908cbdb9bb35c082bc6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001438a80\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-t75v6xkjk6k2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/86bc8f72-988d-4eb5-bc1f-3ed28e9499bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/86bc8f72-988d-4eb5-bc1f-3ed28e9499bb\",\n \"name\": \"86bc8f72-988d-4eb5-bc1f-3ed28e9499bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.070768656Z\",\n \"endTime\": \"2026-08-25T20:07:46.965898216Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001438a80\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-t75v6xkjk6k2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/86bc8f72-988d-4eb5-bc1f-3ed28e9499bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/86bc8f72-988d-4eb5-bc1f-3ed28e9499bb\",\n \"name\": \"86bc8f72-988d-4eb5-bc1f-3ed28e9499bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:38.070768656Z\",\n \"endTime\": \"2026-08-25T20:07:46.965898216Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6f2c85018b6adfe5790bcec137d5c4546786bdf0519ebd2500b8c56742240ee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"05e9d38ab4a3aaad4a0b5371ed4af41ae1885a11dde78c40b966902d6bf39c3e","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c72900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8h5h4xz7mbfk\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"name\": \"54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.60892922Z\",\n \"endTime\": \"2026-08-25T20:07:46.977733317Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c72900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8h5h4xz7mbfk\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"name\": \"54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.60892922Z\",\n \"endTime\": \"2026-08-25T20:07:46.977733317Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7d37e42dadf9ad0ab1f7e1ebf29a1251e6a96a5fba2d7fe226cfc37a206edab2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"23e0583c28d9032ccce1bd36665f177c2f1db717f6e1cfd08f9ea2a16c2c6de9","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00120ece0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-z4sjdz4m7jnd\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80a4dc76-31ed-4b48-9221-629f6eb83aae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80a4dc76-31ed-4b48-9221-629f6eb83aae\",\n \"name\": \"80a4dc76-31ed-4b48-9221-629f6eb83aae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:49.556820571Z\",\n \"endTime\": \"2026-08-25T20:07:57.185806402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00120ece0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-z4sjdz4m7jnd\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80a4dc76-31ed-4b48-9221-629f6eb83aae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80a4dc76-31ed-4b48-9221-629f6eb83aae\",\n \"name\": \"80a4dc76-31ed-4b48-9221-629f6eb83aae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:49.556820571Z\",\n \"endTime\": \"2026-08-25T20:07:57.185806402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ec88746e594b1fba41717e1ac83878df795c98df15cd769257a7cd123f8e687","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"01e5501399c4ff9330ca8905c22cca453551f730bf7866919a23a1964d735140","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12f40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f5xqwdq9mzqv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"name\": \"b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:19.625710695Z\",\n \"endTime\": \"2026-08-25T20:08:26.92099949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12f40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f5xqwdq9mzqv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"name\": \"b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:19.625710695Z\",\n \"endTime\": \"2026-08-25T20:08:26.92099949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"850c802b6217a6ac3a347a4966698157c69b2298e5b66ee60c0de543590ad4f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"b54b257434a08295e33fb0d9a84522989a2312ce08305e6584ba381ae359da3b","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013de540\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tpg96j5hq2v4\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93d2f23c-0f1e-4485-8d1a-26b5dec539bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93d2f23c-0f1e-4485-8d1a-26b5dec539bd\",\n \"name\": \"93d2f23c-0f1e-4485-8d1a-26b5dec539bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.556043561Z\",\n \"endTime\": \"2026-08-25T20:07:47.030178972Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013de540\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tpg96j5hq2v4\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93d2f23c-0f1e-4485-8d1a-26b5dec539bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93d2f23c-0f1e-4485-8d1a-26b5dec539bd\",\n \"name\": \"93d2f23c-0f1e-4485-8d1a-26b5dec539bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.556043561Z\",\n \"endTime\": \"2026-08-25T20:07:47.030178972Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86cb5ce791b2a70e68fef17459aa38bb06ed1963a13dc795b637d1df6b016bb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"62d830a8043675e8a90f508cb3493ce60a38c6b2ad4c7bb07df6c3de3aa1da2c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c008e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qmfzw8r72h4d\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"name\": \"b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:51:05.668136899Z\",\n \"endTime\": \"2026-08-25T20:10:09.078300161Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c008e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qmfzw8r72h4d\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"name\": \"b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:51:05.668136899Z\",\n \"endTime\": \"2026-08-25T20:10:09.078300161Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b8a30cdb2c5357ce355fd1129c61b9eac3bd62cf535774a8434346dc7c3d63a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"42009cf5afc6cd4364288595dd3fe884cb5f131f6903a745c42078e5d885c516","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efe340\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-l4wc2krdbvh4\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5aecd4b7-85cf-4e18-8564-d865e8f29e90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5aecd4b7-85cf-4e18-8564-d865e8f29e90\",\n \"name\": \"5aecd4b7-85cf-4e18-8564-d865e8f29e90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.102866698Z\",\n \"endTime\": \"2026-08-25T20:07:46.967618738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efe340\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-l4wc2krdbvh4\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5aecd4b7-85cf-4e18-8564-d865e8f29e90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5aecd4b7-85cf-4e18-8564-d865e8f29e90\",\n \"name\": \"5aecd4b7-85cf-4e18-8564-d865e8f29e90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.102866698Z\",\n \"endTime\": \"2026-08-25T20:07:46.967618738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c3d155a33d2ec61c3f3dc3c1da1756ed5dfeda0b3d980de2a3a059bf922fdb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"b13e609f7295dd74da4f9e8e1f1a399934f7c35335c10bd5927666f26ed38ac2","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009be5c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h4jwh427c5nz\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e72d10d1-15c4-4ddd-8ca6-468bce1ada88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e72d10d1-15c4-4ddd-8ca6-468bce1ada88\",\n \"name\": \"e72d10d1-15c4-4ddd-8ca6-468bce1ada88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:50:07.046039939Z\",\n \"endTime\": \"2026-08-25T20:09:09.158318976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009be5c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h4jwh427c5nz\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e72d10d1-15c4-4ddd-8ca6-468bce1ada88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e72d10d1-15c4-4ddd-8ca6-468bce1ada88\",\n \"name\": \"e72d10d1-15c4-4ddd-8ca6-468bce1ada88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:50:07.046039939Z\",\n \"endTime\": \"2026-08-25T20:09:09.158318976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"95aba04031de188bb4c8411268333965fa553e32e99c20954514a6e4de7df14f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"5789fa442d4f8a0f4909b27b633de87680963e17d09512bd15731d2bd37905ee","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9rzs78kw8fst/cluster-hs-pre-dfqxhk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f76220\u003e: \n failed to create HCP cluster cluster-hs-pre-dfqxhk: failed waiting for cluster=\"cluster-hs-pre-dfqxhk\" in resourcegroup=\"rg-hs-pre--9rzs78kw8fst\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0267b7d8-6cdf-4409-aaf1-67a83782a8b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0267b7d8-6cdf-4409-aaf1-67a83782a8b9\",\n \"name\": \"0267b7d8-6cdf-4409-aaf1-67a83782a8b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.345359674Z\",\n \"endTime\": \"2026-08-25T20:07:47.045402567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9rzs78kw8fst/cluster-hs-pre-dfqxhk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f76220\u003e: \n failed to create HCP cluster cluster-hs-pre-dfqxhk: failed waiting for cluster=\"cluster-hs-pre-dfqxhk\" in resourcegroup=\"rg-hs-pre--9rzs78kw8fst\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0267b7d8-6cdf-4409-aaf1-67a83782a8b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0267b7d8-6cdf-4409-aaf1-67a83782a8b9\",\n \"name\": \"0267b7d8-6cdf-4409-aaf1-67a83782a8b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.345359674Z\",\n \"endTime\": \"2026-08-25T20:07:47.045402567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9aac2e463b223e90948bb1a94a1ae1f21b266972c83f47ddec328a21145ce29c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"68545017c2b3bd81d0621bcf01c19e0de3848f03893c798a6c2aef8169f05af6","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001ccb410\u003e: \n GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e07f0e45-1eb4-4a7b-9286-54bc68f15713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e07f0e45-1eb4-4a7b-9286-54bc68f15713\",\n \"name\": \"e07f0e45-1eb4-4a7b-9286-54bc68f15713\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:32.516827913Z\",\n \"endTime\": \"2026-08-25T20:08:39.114705542Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001ccb410\u003e: \n GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e07f0e45-1eb4-4a7b-9286-54bc68f15713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e07f0e45-1eb4-4a7b-9286-54bc68f15713\",\n \"name\": \"e07f0e45-1eb4-4a7b-9286-54bc68f15713\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:49:32.516827913Z\",\n \"endTime\": \"2026-08-25T20:08:39.114705542Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a81a5f5e1071a4d9da7d8b9e462b521c24f74e0c21ba4d808338303cdeccca48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"0b9d16275b3d7ff10a538398088cfd3b34d1f81c4cb68201208e8e4940acdb66","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8e5e0\u003e: \n failed to create HCP cluster np-downgrade-x827rx: failed waiting for cluster=\"np-downgrade-x827rx\" in resourcegroup=\"rg-np-downgrade-x827rx-59zprw89nzrh\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"name\": \"601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:49.295862053Z\",\n \"endTime\": \"2026-08-25T20:07:56.866737842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8e5e0\u003e: \n failed to create HCP cluster np-downgrade-x827rx: failed waiting for cluster=\"np-downgrade-x827rx\" in resourcegroup=\"rg-np-downgrade-x827rx-59zprw89nzrh\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"name\": \"601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:49.295862053Z\",\n \"endTime\": \"2026-08-25T20:07:56.866737842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"acc65c2fcdd3d4ce4972a5a3b369b29c92b4277e4098fbe9d0d33f6f509113c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"alert","signature_id":"c8e2dae791d9eee6c2dfe89ff5eb24b232035b0f6a8f9722f2f09c4da4c46270","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T20:01:01Z\n Ended: 2026-08-25T20:16:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0886784-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdq6r93avnmkufpa2dlhkdn5bhh4958-c6q7t8b4b8e6m3v\", pod=\"router-7dffdfb86b-r9dqt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdq6r93avnmkufpa2dlhkdn5bhh4958-c6q7t8b4b8e6m3v/router-7dffdfb86b-r9dqt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T20:01:01Z\n Ended: 2026-08-25T20:17:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0886784-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdq785qenoju9gska8l0aimh5kanj73-r9g0o6t5l6s9a3i\", pod=\"router-87ddfb859-rc8wd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdq785qenoju9gska8l0aimh5kanj73-r9g0o6t5l6s9a3i/router-87ddfb859-rc8wd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"acd8cc430a9c04ebfa944604b4ee7cf76349d60dfb849c7e45012ead79955927","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"c278280dc667ba84567309478370f71e50ba651e2a8a2399ea8b30d7333b71fe","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-7nz4q4 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098e220\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-7nz4q4: failed waiting for cluster=\"np-version-upgrade-cluster-7nz4q4\" in resourcegroup=\"rg-np-version-upgrade-7nz4q4-4zm7chgkt2b9\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1dcec3f-565b-4652-99f1-1c561d01e568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1dcec3f-565b-4652-99f1-1c561d01e568\",\n \"name\": \"d1dcec3f-565b-4652-99f1-1c561d01e568\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:57.347603875Z\",\n \"endTime\": \"2026-08-25T20:07:57.165742948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-7nz4q4 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098e220\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-7nz4q4: failed waiting for cluster=\"np-version-upgrade-cluster-7nz4q4\" in resourcegroup=\"rg-np-version-upgrade-7nz4q4-4zm7chgkt2b9\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1dcec3f-565b-4652-99f1-1c561d01e568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1dcec3f-565b-4652-99f1-1c561d01e568\",\n \"name\": \"d1dcec3f-565b-4652-99f1-1c561d01e568\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:57.347603875Z\",\n \"endTime\": \"2026-08-25T20:07:57.165742948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b65241947ae1fcbf53c1071f67f718a479b520e3e61e4e294c985292fc2e9e4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"15594220caf851c71a98818822ec8afd99496a6032084cc875914f18e55fca35","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-px8pg2 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe6120\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-px8pg2: failed waiting for cluster=\"np-version-upgrade-cluster-px8pg2\" in resourcegroup=\"rg-np-version-upgrade-px8pg2-8p6qlpbvcfl2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61fd5a3-40c1-4dcb-9d76-17e9f07075d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61fd5a3-40c1-4dcb-9d76-17e9f07075d8\",\n \"name\": \"c61fd5a3-40c1-4dcb-9d76-17e9f07075d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:48.623150832Z\",\n \"endTime\": \"2026-08-25T20:07:57.01088836Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-px8pg2 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe6120\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-px8pg2: failed waiting for cluster=\"np-version-upgrade-cluster-px8pg2\" in resourcegroup=\"rg-np-version-upgrade-px8pg2-8p6qlpbvcfl2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61fd5a3-40c1-4dcb-9d76-17e9f07075d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61fd5a3-40c1-4dcb-9d76-17e9f07075d8\",\n \"name\": \"c61fd5a3-40c1-4dcb-9d76-17e9f07075d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:48.623150832Z\",\n \"endTime\": \"2026-08-25T20:07:57.01088836Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b758f30e4be0cf93eae0f43ebf98e6e2c23d7a27422a6e7f9fb3569106ec5205","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"b93a6245ab5010a92efa1e75bc52e899c5a32aee1aa888716b9a188639738a46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"0ddc6a1225a6aa5e4af4f5f0cac4a1b2429896b3b2b8ecd24544d0bdee9d655e","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102c260\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-2hpd7wnk77wj\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f42189bb-3119-4068-9e92-517e4dc57040\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f42189bb-3119-4068-9e92-517e4dc57040\",\n \"name\": \"f42189bb-3119-4068-9e92-517e4dc57040\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:58.883765914Z\",\n \"endTime\": \"2026-08-25T20:08:07.333109761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102c260\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-2hpd7wnk77wj\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f42189bb-3119-4068-9e92-517e4dc57040\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f42189bb-3119-4068-9e92-517e4dc57040\",\n \"name\": \"f42189bb-3119-4068-9e92-517e4dc57040\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:58.883765914Z\",\n \"endTime\": \"2026-08-25T20:08:07.333109761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c2c56ec94c66614729ab471ce4a71aecfb2df053875545ce63a5d8708fe6602e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"53f56986e4035859cdad5301f526d2a5f8fccefb68779c8f991838db02bdf8d5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-tvpgrjv7j552/cluster-candidate-4-22-hrxdt8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001114b40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hrxdt8: failed waiting for cluster=\"cluster-candidate-4-22-hrxdt8\" in resourcegroup=\"rg-candidate-4-22-tvpgrjv7j552\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9a1be9b-c84b-4fef-b784-4e7014774427\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9a1be9b-c84b-4fef-b784-4e7014774427\",\n \"name\": \"a9a1be9b-c84b-4fef-b784-4e7014774427\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:36.13304517Z\",\n \"endTime\": \"2026-08-25T20:07:36.918914574Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-tvpgrjv7j552/cluster-candidate-4-22-hrxdt8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001114b40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hrxdt8: failed waiting for cluster=\"cluster-candidate-4-22-hrxdt8\" in resourcegroup=\"rg-candidate-4-22-tvpgrjv7j552\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9a1be9b-c84b-4fef-b784-4e7014774427\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9a1be9b-c84b-4fef-b784-4e7014774427\",\n \"name\": \"a9a1be9b-c84b-4fef-b784-4e7014774427\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:36.13304517Z\",\n \"endTime\": \"2026-08-25T20:07:36.918914574Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c32d203d9a37e4f46cc51f6bebb56afb1568066775c6e53a0eede877d33d8b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"b933bdf4882baaf62138a22d1375d2afc64afb311d072ff04469393bbe10f8d1","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd86e0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-tvxq99wx8x9r\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6ab4e50-814c-4a52-ac07-b08cd72c343c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6ab4e50-814c-4a52-ac07-b08cd72c343c\",\n \"name\": \"d6ab4e50-814c-4a52-ac07-b08cd72c343c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.989923089Z\",\n \"endTime\": \"2026-08-25T20:07:47.059000808Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd86e0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-tvxq99wx8x9r\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6ab4e50-814c-4a52-ac07-b08cd72c343c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6ab4e50-814c-4a52-ac07-b08cd72c343c\",\n \"name\": \"d6ab4e50-814c-4a52-ac07-b08cd72c343c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.989923089Z\",\n \"endTime\": \"2026-08-25T20:07:47.059000808Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d47583b1beeaad0b96c5556f4da060ddc76d8737c5948aef047943e9f37149b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"b67939caaf0419b44e1276fe638732fc70a0ba57b9c847d29eed62ccca5bd266","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b5a4a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-k5gldl6ws9vz\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62b7beaa-11ff-4f9a-9cb2-c83a0926be3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62b7beaa-11ff-4f9a-9cb2-c83a0926be3c\",\n \"name\": \"62b7beaa-11ff-4f9a-9cb2-c83a0926be3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:28.014637072Z\",\n \"endTime\": \"2026-08-25T20:07:37.031087079Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b5a4a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-k5gldl6ws9vz\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62b7beaa-11ff-4f9a-9cb2-c83a0926be3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62b7beaa-11ff-4f9a-9cb2-c83a0926be3c\",\n \"name\": \"62b7beaa-11ff-4f9a-9cb2-c83a0926be3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:28.014637072Z\",\n \"endTime\": \"2026-08-25T20:07:37.031087079Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d5ccd417bdccc7d5dfe1611614814a5666904e2e18d4ed4e4c476d8e8a1f9d7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"bf05905ae686b69af2afa5478616503e24344e881559bb70f69d4c9570b0f7a8","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-w6l5xk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14340\u003e: \n failed to create HCP cluster np-noedge-w6l5xk: failed waiting for cluster=\"np-noedge-w6l5xk\" in resourcegroup=\"rg-np-noedge-w6l5xk-t6927982vnd2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fa8e0f0-47db-454e-ac19-85245245b727\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fa8e0f0-47db-454e-ac19-85245245b727\",\n \"name\": \"7fa8e0f0-47db-454e-ac19-85245245b727\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.644645718Z\",\n \"endTime\": \"2026-08-25T20:07:57.155379093Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-w6l5xk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14340\u003e: \n failed to create HCP cluster np-noedge-w6l5xk: failed waiting for cluster=\"np-noedge-w6l5xk\" in resourcegroup=\"rg-np-noedge-w6l5xk-t6927982vnd2\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fa8e0f0-47db-454e-ac19-85245245b727\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fa8e0f0-47db-454e-ac19-85245245b727\",\n \"name\": \"7fa8e0f0-47db-454e-ac19-85245245b727\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.644645718Z\",\n \"endTime\": \"2026-08-25T20:07:57.155379093Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d69b1623d62d66d2c2c230cb81e9b90529917c7caad3672747c2d33f21364a5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"6bdc706667de6fd9046a69b69bd20920f987c7f6feff220cb363a388a2ebc362","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a1a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-s9sfpjmwlbkx\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89a068eb-ad3e-4b4e-a266-d184a54fc033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89a068eb-ad3e-4b4e-a266-d184a54fc033\",\n \"name\": \"89a068eb-ad3e-4b4e-a266-d184a54fc033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.028975136Z\",\n \"endTime\": \"2026-08-25T20:07:57.052379873Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a1a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-s9sfpjmwlbkx\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89a068eb-ad3e-4b4e-a266-d184a54fc033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89a068eb-ad3e-4b4e-a266-d184a54fc033\",\n \"name\": \"89a068eb-ad3e-4b4e-a266-d184a54fc033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.028975136Z\",\n \"endTime\": \"2026-08-25T20:07:57.052379873Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"da94cc1c98d025cfafd87a5cb6c92329bebac30ddd471bb250fc36d6e1b87fad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"da7749385b9e21b7d26b42f57364958d539d80f06df4643564a35c6623b5bb10","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-xkn9cmt65svt/cluster-candidate-5-0-m4rmtp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98e20\u003e: \n failed to create HCP cluster cluster-candidate-5-0-m4rmtp: failed waiting for cluster=\"cluster-candidate-5-0-m4rmtp\" in resourcegroup=\"rg-candidate-5-0-xkn9cmt65svt\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c5e0d50-ff9e-465a-85dc-c7ed5412d833\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c5e0d50-ff9e-465a-85dc-c7ed5412d833\",\n \"name\": \"0c5e0d50-ff9e-465a-85dc-c7ed5412d833\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.346974834Z\",\n \"endTime\": \"2026-08-25T20:07:46.987596462Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-xkn9cmt65svt/cluster-candidate-5-0-m4rmtp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98e20\u003e: \n failed to create HCP cluster cluster-candidate-5-0-m4rmtp: failed waiting for cluster=\"cluster-candidate-5-0-m4rmtp\" in resourcegroup=\"rg-candidate-5-0-xkn9cmt65svt\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c5e0d50-ff9e-465a-85dc-c7ed5412d833\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c5e0d50-ff9e-465a-85dc-c7ed5412d833\",\n \"name\": \"0c5e0d50-ff9e-465a-85dc-c7ed5412d833\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.346974834Z\",\n \"endTime\": \"2026-08-25T20:07:46.987596462Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e07adfcd8c4daabbf27fab6917fddc628713c370594d5530477bbdffb085dc9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"702d137fa1b1cc6cdf5e5a67bee5dcd22b5b57ed6578939ac00222e0d40e5187","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-4zgjqt\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f36140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-4zgjqt: failed waiting for cluster=\"cluster-zstream-4-20-4zgjqt\" in resourcegroup=\"rg-zstream-upgrade-4-20-xbl5pnsrg4h5\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d7de8f4-be99-4a12-a44f-810f1b793f24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d7de8f4-be99-4a12-a44f-810f1b793f24\",\n \"name\": \"9d7de8f4-be99-4a12-a44f-810f1b793f24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.352958865Z\",\n \"endTime\": \"2026-08-25T20:07:57.041102903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-4zgjqt\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f36140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-4zgjqt: failed waiting for cluster=\"cluster-zstream-4-20-4zgjqt\" in resourcegroup=\"rg-zstream-upgrade-4-20-xbl5pnsrg4h5\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d7de8f4-be99-4a12-a44f-810f1b793f24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d7de8f4-be99-4a12-a44f-810f1b793f24\",\n \"name\": \"9d7de8f4-be99-4a12-a44f-810f1b793f24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.352958865Z\",\n \"endTime\": \"2026-08-25T20:07:57.041102903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e164bfa842fc42b6f01300e6bf5d7b495cd14505800cd56ef2b155856d809ab7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"2b7bfba4308decd6036acc7bc3a1df135b34ae217fab985b49db26b737a0d3a3","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e6c260\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-k29pdvjpddtt\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ebb70e6-c41a-4dfd-b5e7-fbc68001b621\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ebb70e6-c41a-4dfd-b5e7-fbc68001b621\",\n \"name\": \"3ebb70e6-c41a-4dfd-b5e7-fbc68001b621\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:35.619346314Z\",\n \"endTime\": \"2026-08-25T20:07:36.964636575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e6c260\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-k29pdvjpddtt\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ebb70e6-c41a-4dfd-b5e7-fbc68001b621\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ebb70e6-c41a-4dfd-b5e7-fbc68001b621\",\n \"name\": \"3ebb70e6-c41a-4dfd-b5e7-fbc68001b621\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:35.619346314Z\",\n \"endTime\": \"2026-08-25T20:07:36.964636575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ef3f5e36d9626593f94e71afe4bbb9589f40d9d0a304219cc0951ff9f1dfb0b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"9bc356d5e4cf33cadfb9cc3dddabf59890f14d30c9b21e128d830f5bcd9b3092","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000958b60\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pdjrtfw6d46q\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd1cc1a0-d45b-4741-8daf-968e1a7ce6e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd1cc1a0-d45b-4741-8daf-968e1a7ce6e1\",\n \"name\": \"cd1cc1a0-d45b-4741-8daf-968e1a7ce6e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.200718331Z\",\n \"endTime\": \"2026-08-25T20:07:46.996110266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000958b60\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pdjrtfw6d46q\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd1cc1a0-d45b-4741-8daf-968e1a7ce6e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd1cc1a0-d45b-4741-8daf-968e1a7ce6e1\",\n \"name\": \"cd1cc1a0-d45b-4741-8daf-968e1a7ce6e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:46.200718331Z\",\n \"endTime\": \"2026-08-25T20:07:46.996110266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f237ea500e57e9607c2d1835e0df4baf547cb0b6eb7fb46ec965b92775bca174","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"dee87de74156ea23b8b3520f533ef9190f72b7babcd20bcb8e2cf2a94d10dcdb","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a66340\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-w6n95hx6cs8g\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9212c53c-0e9b-4c1c-8b8e-d1f695990af3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9212c53c-0e9b-4c1c-8b8e-d1f695990af3\",\n \"name\": \"9212c53c-0e9b-4c1c-8b8e-d1f695990af3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:36.47765205Z\",\n \"endTime\": \"2026-08-25T20:07:37.081476006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a66340\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-w6n95hx6cs8g\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9212c53c-0e9b-4c1c-8b8e-d1f695990af3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9212c53c-0e9b-4c1c-8b8e-d1f695990af3\",\n \"name\": \"9212c53c-0e9b-4c1c-8b8e-d1f695990af3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:36.47765205Z\",\n \"endTime\": \"2026-08-25T20:07:37.081476006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7a2eb160500ff27207fcdd7f47aec74cc2c84c6a698deaf80e146246fbcb089","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"d693677dcc2436c081b021447374afa98eb7158ee0d3dec68e19d233b84b5470","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019c2120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-f4v8vdbbpdlg\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7aedcacf-cef9-4283-afd0-2b0927e5e241\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7aedcacf-cef9-4283-afd0-2b0927e5e241\",\n \"name\": \"7aedcacf-cef9-4283-afd0-2b0927e5e241\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.725696623Z\",\n \"endTime\": \"2026-08-25T20:07:57.176827572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019c2120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-f4v8vdbbpdlg\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7aedcacf-cef9-4283-afd0-2b0927e5e241\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7aedcacf-cef9-4283-afd0-2b0927e5e241\",\n \"name\": \"7aedcacf-cef9-4283-afd0-2b0927e5e241\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:56.725696623Z\",\n \"endTime\": \"2026-08-25T20:07:57.176827572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fad7ee6c719bccf005c029d76cde77f29df03cb71b5d07cdd84fc9cfc7cfccd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"dfa42aba5c4cc2cef7d699b8c65ba25382242bb42c7e266ef350ef67bc80df08","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e12280\u003e: \n failed to create HCP cluster ea-listq5848k: failed waiting for cluster=\"ea-listq5848k\" in resourcegroup=\"ea-list-nssqzwnb56dl\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aa1464e-2383-4ca2-867b-7560881f36fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aa1464e-2383-4ca2-867b-7560881f36fd\",\n \"name\": \"0aa1464e-2383-4ca2-867b-7560881f36fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.039045751Z\",\n \"endTime\": \"2026-08-25T20:07:57.125257395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e12280\u003e: \n failed to create HCP cluster ea-listq5848k: failed waiting for cluster=\"ea-listq5848k\" in resourcegroup=\"ea-list-nssqzwnb56dl\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aa1464e-2383-4ca2-867b-7560881f36fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aa1464e-2383-4ca2-867b-7560881f36fd\",\n \"name\": \"0aa1464e-2383-4ca2-867b-7560881f36fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:48:47.039045751Z\",\n \"endTime\": \"2026-08-25T20:07:57.125257395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fd95230971973d47b7f48b732aba7b0c0136249322293b24704c235d86a29234","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":54,"failure_pattern_ids":["040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":54,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6683,"pr_state":"open","pr_sha":"817f0c148448a8095f1eeb621296577f82351c40","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:23:10Z"},"failed_at":["alert","e2e"],"failed_test_count":24,"occurrences":[{"row_id":"0bdfc30bf71bd297587a88bd1b4e1609e52d5ad1603faf4d62b18b78c5b37473","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"b0b6ea785c6adb804d2ab49b2e9c929c0658e46f9052aac84f220ac1f23f4ce1","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abe4c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b7hpj2n7vp57\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\",\n \"name\": \"6613b466-27ca-4523-a803-29e6469bc399\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:41.448636527Z\",\n \"endTime\": \"2026-08-25T19:39:42.477039993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abe4c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b7hpj2n7vp57\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\",\n \"name\": \"6613b466-27ca-4523-a803-29e6469bc399\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:41.448636527Z\",\n \"endTime\": \"2026-08-25T19:39:42.477039993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0dead54079fb2a63e3cfb928d33740b0de2406d75056e5f832fa5022d7a10595","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"bf78b2adb206b76ebe7b562c8f5d0dcff4cb6f69f3835690af34a77d874060a1","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019ed80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lxqrtnj4sd4l\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"name\": \"6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:28.693998542Z\",\n \"endTime\": \"2026-08-25T19:39:32.413356519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019ed80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lxqrtnj4sd4l\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"name\": \"6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:28.693998542Z\",\n \"endTime\": \"2026-08-25T19:39:32.413356519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"69fd70ce82cd9e2e8d6772c753f2a02b08d82ff9d1871bc660a3545c674182f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"7a9b3449c6776fb39f3e7d85c2865848c6849a9fed0d320c413b71edcfe0b968","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8jnkhgqhx7vj/cluster-hs-pre-gcwzlp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ee120\u003e: \n failed to create HCP cluster cluster-hs-pre-gcwzlp: failed waiting for cluster=\"cluster-hs-pre-gcwzlp\" in resourcegroup=\"rg-hs-pre--8jnkhgqhx7vj\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"name\": \"aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:58.14501483Z\",\n \"endTime\": \"2026-08-25T19:38:02.43879336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8jnkhgqhx7vj/cluster-hs-pre-gcwzlp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ee120\u003e: \n failed to create HCP cluster cluster-hs-pre-gcwzlp: failed waiting for cluster=\"cluster-hs-pre-gcwzlp\" in resourcegroup=\"rg-hs-pre--8jnkhgqhx7vj\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"name\": \"aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:58.14501483Z\",\n \"endTime\": \"2026-08-25T19:38:02.43879336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d188724e2ad8b2ff0009817ae8838a4d0cce4a292b1f33e9f843e12ff24e190c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a0a83ae0d006b1e5515fa9ca2a57ddf53bcb2f16023f115bf32a8e19b0229c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"3a165bfaf5767e237e81c7162c0c48f850249eb7416a5043163724820b2a6d91","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019fc320\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-pvw94n8jrfrs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"name\": \"17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:50.270808871Z\",\n \"endTime\": \"2026-08-25T19:37:52.396630467Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, csi-snapshot-controller-operator, packageserver, machine-approver, cluster-policy-controller, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, ingress-operator, catalog-operator, olm-operator, community-operators-catalog, dns-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019fc320\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-pvw94n8jrfrs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"name\": \"17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:50.270808871Z\",\n \"endTime\": \"2026-08-25T19:37:52.396630467Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, csi-snapshot-controller-operator, packageserver, machine-approver, cluster-policy-controller, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, ingress-operator, catalog-operator, olm-operator, community-operators-catalog, dns-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e258b478ac9644b09eb20a3a01941c7945db5504ffd2b81616ca149fbb4626ea","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a8d27a9879f7f735abdcc4c2b59792d642c41d71afb0075face23da186986ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"8ad26a920569e563149923a5bcd439d71e32d4914bb330af2790f80b14322048","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a1a0\u003e: \n failed to create HCP cluster ea-listmdvtnb: failed waiting for cluster=\"ea-listmdvtnb\" in resourcegroup=\"ea-list-fwskcqw558j7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"name\": \"09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:23.028754386Z\",\n \"endTime\": \"2026-08-25T19:39:32.394595575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a1a0\u003e: \n failed to create HCP cluster ea-listmdvtnb: failed waiting for cluster=\"ea-listmdvtnb\" in resourcegroup=\"ea-list-fwskcqw558j7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"name\": \"09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:23.028754386Z\",\n \"endTime\": \"2026-08-25T19:39:32.394595575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70b055e56f87fd1600299d44cf588275d0c035e88106f89236f14dc564a465c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"bc247a67bb8f719d23e560a19b4f04763bcfe81ecbf03218468d9c66f357d587","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001336360\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4mq79tbwfj5b\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"name\": \"a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:48.128240196Z\",\n \"endTime\": \"2026-08-25T19:37:52.295117327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001336360\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4mq79tbwfj5b\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"name\": \"a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:48.128240196Z\",\n \"endTime\": \"2026-08-25T19:37:52.295117327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"442531230d6bcd61fe689ddb6dd1199a972aefc3d6bea05b5c34201c97dcd098","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"772b7f73903b21debd5e72cfffa74e762d8e2727ad26125843209de53225b5dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"c9efb511f1cb5ecb77052682b34bef74a06a5d809c163ae9d53ba77fc8559f19","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-nhz9z7\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8a2a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-nhz9z7: failed waiting for cluster=\"cluster-zstream-4-21-nhz9z7\" in resourcegroup=\"rg-zstream-upgrade-4-21-mm7bccs84p4v\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561e76bb-0caf-4ffd-8bd7-8e2a4007a09a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561e76bb-0caf-4ffd-8bd7-8e2a4007a09a\",\n \"name\": \"561e76bb-0caf-4ffd-8bd7-8e2a4007a09a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:43.538301268Z\",\n \"endTime\": \"2026-08-25T19:41:52.421747743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-nhz9z7\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8a2a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-nhz9z7: failed waiting for cluster=\"cluster-zstream-4-21-nhz9z7\" in resourcegroup=\"rg-zstream-upgrade-4-21-mm7bccs84p4v\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561e76bb-0caf-4ffd-8bd7-8e2a4007a09a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561e76bb-0caf-4ffd-8bd7-8e2a4007a09a\",\n \"name\": \"561e76bb-0caf-4ffd-8bd7-8e2a4007a09a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:43.538301268Z\",\n \"endTime\": \"2026-08-25T19:41:52.421747743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b3fdf9c2dd7df8ae79bf3f74a82b53e3a38d4e34efcc3e5cf09514bec5c8f49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"e7d19e4ec13af075778836b0509e48aae2d3aa918859115e31905f9216f6eda2","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cc300\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-j56jzgdfb84c\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"name\": \"152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:55.074048426Z\",\n \"endTime\": \"2026-08-25T19:42:02.420103182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cc300\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-j56jzgdfb84c\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"name\": \"152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:55.074048426Z\",\n \"endTime\": \"2026-08-25T19:42:02.420103182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d0f63cbf18696135f2b849375c519cf713c57b302ca24c299a93ee60aadbb67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"84131327fae01288ea8861e4a967044b615e020a15959c0d9a099127d951ae13","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddcf40\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-q2q7d86xnzgs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"name\": \"97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:39.021640766Z\",\n \"endTime\": \"2026-08-25T19:37:42.379207389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddcf40\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-q2q7d86xnzgs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"name\": \"97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:39.021640766Z\",\n \"endTime\": \"2026-08-25T19:37:42.379207389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a8b9eff5109f2cc78b2b4b95812104bd6d8fe537f248bb3620a7bea6362a087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"1a10e9f0d0ebd327c61e8e899905b12b28ad26f43e7f89a4aa098b09ef35fc21","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0007ab4d0\u003e: \n GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"name\": \"805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:32.391153149Z\",\n \"endTime\": \"2026-08-25T19:39:32.410048712Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0007ab4d0\u003e: \n GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"name\": \"805512f7-5822-4d4a-b7f2-904c7a51afce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:32.391153149Z\",\n \"endTime\": \"2026-08-25T19:39:32.410048712Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a13fcf77ef1bb55c7aea987da0e0b5f8798ca4ef171e45d4478e2f2cc9865017","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"01abb2e6b898b7ccf9ba858e60ea62a7872111a48241f292195140bc087ead89","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e400\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mvgcw5tqmrmq\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"name\": \"8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:00.213672088Z\",\n \"endTime\": \"2026-08-25T19:38:02.372107389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, olm-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, packageserver, cluster-node-tuning-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e400\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mvgcw5tqmrmq\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"name\": \"8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:00.213672088Z\",\n \"endTime\": \"2026-08-25T19:38:02.372107389Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, olm-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, packageserver, cluster-node-tuning-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f135b58213daed6ea6fbc2013a314b004e48b63448926884614927be5a42fc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aea745361d0b80cfae6fb4d24b864b5ee06fb1978f39d0250d964040ec972683","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"alert","signature_id":"57c5ec60d7b5f08bc41aec5dff584edf947ed9bf0d23769b021eb635a33b3d21","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:49:51Z\n Ended: 2026-08-25T19:57:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8405376-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8405376-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"b8f0bf8a6a175313144f5279447ea97f7129e726e904feb8f1719517e7dd7e04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"6ae7c340a3a6185150f01fedd02fdfde8bcb2b6e79e7c90419ba5cf8f2b0a0c6","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc8b80\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-r67n76nc45fp\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"name\": \"f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.043336048Z\",\n \"endTime\": \"2026-08-25T19:39:32.419104707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc8b80\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-r67n76nc45fp\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"name\": \"f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.043336048Z\",\n \"endTime\": \"2026-08-25T19:39:32.419104707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b9a77bd4bae1dbb3b2b60b9218a9b9e0ed7b0432f6fe6fd597b54dfc9328bd31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"bdffa9ec41658c3c0b2fef63b1f22f6a6742e6c4e837e0fea114368b4612c51e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d808a0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m46xgd2l9dwf\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"name\": \"93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.65358047Z\",\n \"endTime\": \"2026-08-25T19:39:22.258875158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d808a0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m46xgd2l9dwf\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"name\": \"93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.65358047Z\",\n \"endTime\": \"2026-08-25T19:39:22.258875158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb7fd09623e99d3db04fac97780bf93a3e41da64d571f0d249d26bce9cf66bb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"569936c5ee5c9e4cdd5de51f80b6ee2ec197ed73598a8d190cbd9e81f0f34f1b","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pqxcnq8z9jpd\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"name\": \"bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.984108593Z\",\n \"endTime\": \"2026-08-25T19:39:32.326305358Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pqxcnq8z9jpd\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"name\": \"bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:31.984108593Z\",\n \"endTime\": \"2026-08-25T19:39:32.326305358Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bc9a850b17173089847035d1717abfaeb29b8ed2ed503bf29c6456ed5b381923","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"32b84312ecc0412cc4b93a9ef15c32fc382121719b6562881ec5a616688de541","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-p6zdptkz52r2/cluster-candidate-4-21-8fkhjq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e62640\u003e: \n failed to create HCP cluster cluster-candidate-4-21-8fkhjq: failed waiting for cluster=\"cluster-candidate-4-21-8fkhjq\" in resourcegroup=\"rg-candidate-4-21-p6zdptkz52r2\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"name\": \"8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:40.508277835Z\",\n \"endTime\": \"2026-08-25T19:39:42.488198943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-p6zdptkz52r2/cluster-candidate-4-21-8fkhjq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e62640\u003e: \n failed to create HCP cluster cluster-candidate-4-21-8fkhjq: failed waiting for cluster=\"cluster-candidate-4-21-8fkhjq\" in resourcegroup=\"rg-candidate-4-21-p6zdptkz52r2\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"name\": \"8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:40.508277835Z\",\n \"endTime\": \"2026-08-25T19:39:42.488198943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fc48aeaa99b8367ce8df90e24c1324426e971a1da3ed42097c5345639e1ee2c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c183ecdaadcab109447689088616e30240ace58de917d175efc41951db95c374","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"91a343c747731a46478c011c994a808882abeec84c6530eb04b1878042920e4e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26160\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q672lrjjt7xx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"name\": \"5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:59.654390099Z\",\n \"endTime\": \"2026-08-25T19:38:02.42081904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26160\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q672lrjjt7xx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"name\": \"5ba6b631-6240-4773-bfa1-ea8d27930011\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:59.654390099Z\",\n \"endTime\": \"2026-08-25T19:38:02.42081904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c1e97a14c673f942d746f1cffb5c3a50cd21c6a835064ccd4e54ad0b62d70ef4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"alert","signature_id":"00cf1cdbbe1c25edacd565cc3dbeb30b1fc906ca4e493b4125b891e5503ffe76","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 44 time(s) (4 unknown, 40 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:36:34Z\n Ended: 2026-08-25T19:42:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppr507knoluq4b6simih0av16epoc-p8e6x0p1e8y3d3q\", pod=\"cluster-network-operator-7fc64bc864-z8m58\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppr507knoluq4b6simih0av16epoc-p8e6x0p1e8y3d3q/cluster-network-operator-7fc64bc864-z8m58 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T19:37:33Z\n Ended: 2026-08-25T19:43:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb\", pod=\"cluster-network-operator-bc9698665-b8c6g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb/cluster-network-operator-bc9698665-b8c6g has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T19:37:34Z\n Ended: 2026-08-25T19:43:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppt1p2lo3ijsep2g5q057lthm1bnq-y8l2t6b7a3x5m5p\", pod=\"ingress-operator-9dfd956c9-slrm6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppt1p2lo3ijsep2g5q057lthm1bnq-y8l2t6b7a3x5m5p/ingress-operator-9dfd956c9-slrm6 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T19:37:34Z\n Ended: 2026-08-25T19:43:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb\", pod=\"ingress-operator-6f9986dfc-xd7sz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb/ingress-operator-6f9986dfc-xd7sz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e3bb32169ad916a16310d05cfb72cfdcc2d3738716954c6ad242d8a0739c783b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"d496820b37cab7fa1ea7b0c47e837d533327b2947405aa862fe6cf0becda824e","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101a160\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qmzd7vdsthg7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"name\": \"d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:20.481967932Z\",\n \"endTime\": \"2026-08-25T19:39:22.436659526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101a160\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qmzd7vdsthg7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"name\": \"d1cf37f1-e335-4a31-8459-8927106e9275\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:20.481967932Z\",\n \"endTime\": \"2026-08-25T19:39:22.436659526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72de96d5fccd2e929d86c234bb6122fb3e96395118e1ddf6c2c21fc7fb224f37","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e87b9eee0e0eef8e542b9f214b26271aafe0103af343373e23e27cd982aa54b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"b76793c96776650aa6c2c9acf626989ef5411551675fc33c8a57f8cf74578d78","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2c400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v5fxchbb2prs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"name\": \"561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:21:13.116412138Z\",\n \"endTime\": \"2026-08-25T19:40:22.435169961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2c400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v5fxchbb2prs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"name\": \"561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:21:13.116412138Z\",\n \"endTime\": \"2026-08-25T19:40:22.435169961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eb1d493091f6edf10bca4392ae62113b462fe45fb691865c35ee688080633f3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"a2fca96d5c30f50d48abd3d91900db1407214ad4c57c888c781660c67cdbdfbe","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-tlrb8m with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a7a5c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-tlrb8m: failed waiting for cluster=\"np-version-upgrade-cluster-tlrb8m\" in resourcegroup=\"rg-np-version-upgrade-tlrb8m-64c6wd9hqbps\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"name\": \"1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:26.310363981Z\",\n \"endTime\": \"2026-08-25T19:41:32.435006844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-tlrb8m with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a7a5c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-tlrb8m: failed waiting for cluster=\"np-version-upgrade-cluster-tlrb8m\" in resourcegroup=\"rg-np-version-upgrade-tlrb8m-64c6wd9hqbps\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"name\": \"1769c72e-78e8-48ce-b214-b5a858f938c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:26.310363981Z\",\n \"endTime\": \"2026-08-25T19:41:32.435006844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f1f04d78c2f448bd84e89fea57dfe6a2d490a3201fe3074eb5f961cd53bf7d09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"874e564fd64d9955e95921340933ec68b79b24e1dc50ad4f794673936e060215","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-m94cqb294gjz/cluster-candidate-5-0-cd7r5j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd6160\u003e: \n failed to create HCP cluster cluster-candidate-5-0-cd7r5j: failed waiting for cluster=\"cluster-candidate-5-0-cd7r5j\" in resourcegroup=\"rg-candidate-5-0-m94cqb294gjz\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"name\": \"f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:32.285332628Z\",\n \"endTime\": \"2026-08-25T19:41:32.407207308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-m94cqb294gjz/cluster-candidate-5-0-cd7r5j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd6160\u003e: \n failed to create HCP cluster cluster-candidate-5-0-cd7r5j: failed waiting for cluster=\"cluster-candidate-5-0-cd7r5j\" in resourcegroup=\"rg-candidate-5-0-m94cqb294gjz\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"name\": \"f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:22:32.285332628Z\",\n \"endTime\": \"2026-08-25T19:41:32.407207308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc90ab7bd0cdefbc91c7294c629728843756f72a115633a895687212ddeaac87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"383f94fc81a983b134e7f8347429995df9c16fd5e3b4d726b4e05d385fc27709","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2b40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gvqsnvnk8pkx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"name\": \"60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.389463971Z\",\n \"endTime\": \"2026-08-25T19:39:22.423838537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2b40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gvqsnvnk8pkx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"name\": \"60298cc3-cc25-45ff-b759-a7365c10ec56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:21.389463971Z\",\n \"endTime\": \"2026-08-25T19:39:22.423838537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fca1db9aed5b1120ce6bfb571a7e1f38e9cef7f671430d8fcd6e8a2c860e87b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","failed_at":"e2e","signature_id":"6873e149feb92978c0e6c51d500f8132748dcf6414954a9cbceb9d163428d9a0","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c0e640\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58h6tmctnmgt\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"name\": \"d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:02.364791294Z\",\n \"endTime\": \"2026-08-25T19:38:02.432181787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c0e640\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58h6tmctnmgt\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"name\": \"d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:02.364791294Z\",\n \"endTime\": \"2026-08-25T19:38:02.432181787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":24,"failure_pattern_ids":["0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","442531230d6bcd61fe689ddb6dd1199a972aefc3d6bea05b5c34201c97dcd098","6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","72de96d5fccd2e929d86c234bb6122fb3e96395118e1ddf6c2c21fc7fb224f37","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","d188724e2ad8b2ff0009817ae8838a4d0cce4a292b1f33e9f843e12ff24e190c","da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","e258b478ac9644b09eb20a3a01941c7945db5504ffd2b81616ca149fbb4626ea","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f135b58213daed6ea6fbc2013a314b004e48b63448926884614927be5a42fc0d","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","fc48aeaa99b8367ce8df90e24c1324426e971a1da3ed42097c5345639e1ee2c0"],"matched_occurrences":24,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6529,"pr_state":"open","pr_sha":"09ab1aa72405f5244d5d05015c11621d9cbf759a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:21:37Z"},"failed_at":["alert","e2e"],"failed_test_count":12,"occurrences":[{"row_id":"42fcc0d9a4468520560bdbe5843b4d483b2a1a0752eacccebfd4859bd143b944","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"alert","signature_id":"670def82337f4e93346e218bb912d2f5102dda3d4e354be6f7a3ae52af837b74","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 15 time(s) (3 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:38:10Z\n Ended: 2026-08-25T19:44:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6756224-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdprcahq51n0d0h5krenqtt6q04qaja-t5m9k0x6n2j1y0j\", pod=\"cluster-image-registry-operator-6599c97d88-9k277\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdprcahq51n0d0h5krenqtt6q04qaja-t5m9k0x6n2j1y0j/cluster-image-registry-operator-6599c97d88-9k277 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T19:38:10Z\n Ended: 2026-08-25T20:11:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6756224-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpr4u0kflvoq3tr6uonatusgjkqtgb-i7z1k6g5a6y6a9r\", pod=\"router-565466c4ff-x85nf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdpr4u0kflvoq3tr6uonatusgjkqtgb-i7z1k6g5a6y6a9r/router-565466c4ff-x85nf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T19:38:10Z\n Ended: 2026-08-25T19:44:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6756224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpra59hpdand8n0tpnpdkp1smvi0if-b7k4i1r5j7n1v6e\", pod=\"ingress-operator-588fd99d97-qflbj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdpra59hpdand8n0tpnpdkp1smvi0if-b7k4i1r5j7n1v6e/ingress-operator-588fd99d97-qflbj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"73f5506606ee005ba2c074d5158604821010d43045fa85e5ed0af9082888d21b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"7920aacad73a762e7cb4200cc3f935f23f19001cc15e0d371213e2d503e50bf5","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00131c880\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bxjjlgfmc9db\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"name\": \"3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:19.200664013Z\",\n \"endTime\": \"2026-08-25T19:42:23.611070686Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00131c880\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bxjjlgfmc9db\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"name\": \"3a4169eb-2032-480c-8f15-d84b8528bedf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:19.200664013Z\",\n \"endTime\": \"2026-08-25T19:42:23.611070686Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"74ccfa027e5298cdab8e2205711888dd63c996ab233d25b80629864e18aac54f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"acd72cf8b7bb2f45dae55eed6a696c8c2095d677ca82dcc3ccb5ef413f64290e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c92e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8wz885flg8mk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"name\": \"9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:24:58.553705193Z\",\n \"endTime\": \"2026-08-25T19:44:03.525820627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c92e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8wz885flg8mk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"name\": \"9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:24:58.553705193Z\",\n \"endTime\": \"2026-08-25T19:44:03.525820627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"829dc4adf9540aee43370f1832afcdfbb21299104aed900057f11d7435680098","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"c0dcf5c3ac2881b29fcaf6a19b5d8e99b79dd0936f8a80948a5bd717cba76449","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e8bc0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wmnbv4hg75hf\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5253569b-15c5-4d2b-9986-1bbfc4831c80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5253569b-15c5-4d2b-9986-1bbfc4831c80\",\n \"name\": \"5253569b-15c5-4d2b-9986-1bbfc4831c80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:19.118008123Z\",\n \"endTime\": \"2026-08-25T19:42:23.60955276Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e8bc0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wmnbv4hg75hf\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5253569b-15c5-4d2b-9986-1bbfc4831c80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5253569b-15c5-4d2b-9986-1bbfc4831c80\",\n \"name\": \"5253569b-15c5-4d2b-9986-1bbfc4831c80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:19.118008123Z\",\n \"endTime\": \"2026-08-25T19:42:23.60955276Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e379b9b8393f199d434705c3d66af2a249812274abc3a586de303bb14fba23d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"dbe010bd3eef880599620e7f2c75170adabcb8631a1d8b6c5fd42aa908a7aadb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-84dql4hbsnbj/cluster-candidate-5-0-2lchcj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaf8a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-2lchcj: failed waiting for cluster=\"cluster-candidate-5-0-2lchcj\" in resourcegroup=\"rg-candidate-5-0-84dql4hbsnbj\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"name\": \"5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:18.647011306Z\",\n \"endTime\": \"2026-08-25T19:42:23.606424611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-84dql4hbsnbj/cluster-candidate-5-0-2lchcj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaf8a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-2lchcj: failed waiting for cluster=\"cluster-candidate-5-0-2lchcj\" in resourcegroup=\"rg-candidate-5-0-84dql4hbsnbj\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"name\": \"5f71506a-af2f-4b60-985d-00bd566e1074\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:18.647011306Z\",\n \"endTime\": \"2026-08-25T19:42:23.606424611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aea1b07c92a605f53e7aefb66991fd8af7ffdecf2877a9a1c22be4365310abc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"7e61cb838af28069775e4047abe596d948ba92c34b9bf7479496b56271f567b9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-8rp2skrkddck/cluster-candidate-4-21-gtzqwv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001062bc0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gtzqwv: failed waiting for cluster=\"cluster-candidate-4-21-gtzqwv\" in resourcegroup=\"rg-candidate-4-21-8rp2skrkddck\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/21dd8bf0-5d54-4539-b2c9-c280ef5ecac0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/21dd8bf0-5d54-4539-b2c9-c280ef5ecac0\",\n \"name\": \"21dd8bf0-5d54-4539-b2c9-c280ef5ecac0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:28.411842632Z\",\n \"endTime\": \"2026-08-25T19:42:33.584158989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-8rp2skrkddck/cluster-candidate-4-21-gtzqwv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001062bc0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gtzqwv: failed waiting for cluster=\"cluster-candidate-4-21-gtzqwv\" in resourcegroup=\"rg-candidate-4-21-8rp2skrkddck\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/21dd8bf0-5d54-4539-b2c9-c280ef5ecac0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/21dd8bf0-5d54-4539-b2c9-c280ef5ecac0\",\n \"name\": \"21dd8bf0-5d54-4539-b2c9-c280ef5ecac0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:28.411842632Z\",\n \"endTime\": \"2026-08-25T19:42:33.584158989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cae2eb456c165b196e40e88fac6e83ced6b57688d06a2c63d2bc3b7ea3039c0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"2a3c305e2e7b889cc83bd789e19181806498ad35731017cdff9ccc00c9807367","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128a380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7p5xtj25zdgq\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"name\": \"02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.029566085Z\",\n \"endTime\": \"2026-08-25T19:42:33.615773728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128a380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7p5xtj25zdgq\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"name\": \"02c1dd80-6bb6-452a-8773-44a463672ab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.029566085Z\",\n \"endTime\": \"2026-08-25T19:42:33.615773728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d247d244bd7935b3f455367b110bec4fe7c2d5b66e004979746b1b061926099d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"30dad826455fa0938e7cec0e4f031d259f61105244ff995b4e802cf11d598f9a","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e340\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-lh89fqcjwl87\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb38e1dc-22b2-4776-9b04-7ffe614ec247\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb38e1dc-22b2-4776-9b04-7ffe614ec247\",\n \"name\": \"eb38e1dc-22b2-4776-9b04-7ffe614ec247\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:29.211130265Z\",\n \"endTime\": \"2026-08-25T19:42:33.606460539Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e340\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-lh89fqcjwl87\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb38e1dc-22b2-4776-9b04-7ffe614ec247\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb38e1dc-22b2-4776-9b04-7ffe614ec247\",\n \"name\": \"eb38e1dc-22b2-4776-9b04-7ffe614ec247\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:29.211130265Z\",\n \"endTime\": \"2026-08-25T19:42:33.606460539Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e85625ec45be0e89143ce2632051b02ba47d8fd44dd8d99c518a5307d3646496","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"f503b51c49edd527f0ec657181f3029f0c0054cf869d9c05bac64d44223372af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"a5efaacf08081a9c1c8f925d50e651cf29e6bdbb45a3190694979b6ed844c618","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-2gjx56wbhk2l/cluster-candidate-4-22-m6vwk4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b8ea60\u003e: \n failed to create HCP cluster cluster-candidate-4-22-m6vwk4: failed waiting for cluster=\"cluster-candidate-4-22-m6vwk4\" in resourcegroup=\"rg-candidate-4-22-2gjx56wbhk2l\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7780b439-c49a-4b29-b305-38b947d4a1c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7780b439-c49a-4b29-b305-38b947d4a1c1\",\n \"name\": \"7780b439-c49a-4b29-b305-38b947d4a1c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:17.536066943Z\",\n \"endTime\": \"2026-08-25T19:42:23.589759538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-2gjx56wbhk2l/cluster-candidate-4-22-m6vwk4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b8ea60\u003e: \n failed to create HCP cluster cluster-candidate-4-22-m6vwk4: failed waiting for cluster=\"cluster-candidate-4-22-m6vwk4\" in resourcegroup=\"rg-candidate-4-22-2gjx56wbhk2l\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7780b439-c49a-4b29-b305-38b947d4a1c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7780b439-c49a-4b29-b305-38b947d4a1c1\",\n \"name\": \"7780b439-c49a-4b29-b305-38b947d4a1c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:17.536066943Z\",\n \"endTime\": \"2026-08-25T19:42:23.589759538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5e219f206cfc153bb61fd843fef6049e9e6ba55c68140873005e830c5d7468c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"1f94d94c8e70a8e7a8b56a5653939cf4935a72854bc9feda9124071da3fbfe56","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ttf5nhws49r9/cluster-candidate-4-20-fx47tw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a08e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-fx47tw: failed waiting for cluster=\"cluster-candidate-4-20-fx47tw\" in resourcegroup=\"rg-candidate-4-20-ttf5nhws49r9\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"name\": \"7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:17.318805591Z\",\n \"endTime\": \"2026-08-25T19:42:23.583486998Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ttf5nhws49r9/cluster-candidate-4-20-fx47tw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a08e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-fx47tw: failed waiting for cluster=\"cluster-candidate-4-20-fx47tw\" in resourcegroup=\"rg-candidate-4-20-ttf5nhws49r9\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"name\": \"7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:17.318805591Z\",\n \"endTime\": \"2026-08-25T19:42:23.583486998Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f826f67aebcc24ffa81196b18f6a185a21391b57b58e92fc6d86c7b874c70105","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","failed_at":"e2e","signature_id":"2ce22b33b27841f4451ddbf18b57eaa0818348b0deaa93898d8394e7df7e56d0","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f14340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-p6gkdz5j5kdk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"name\": \"3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.431435186Z\",\n \"endTime\": \"2026-08-25T19:42:33.505289846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f14340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-p6gkdz5j5kdk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"name\": \"3cd1f359-f586-4859-b136-632a31d92bfd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:23:27.431435186Z\",\n \"endTime\": \"2026-08-25T19:42:33.505289846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6554,"pr_state":"open","pr_sha":"4e67bc77de1a80644494c4c53d6416cbb8167300","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:14:42Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"1d35cf29beb8e6209b8ef152898461a51e0e72c587566f8b10024c05cb06c350","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"e6e6771b32f676dac7dd06703c5d639fb506b0db753539a9e145282b66c90963","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-wwz9mh at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e6c0\u003e: \n failed to create HCP cluster np-skip-minor-wwz9mh: failed waiting for cluster=\"np-skip-minor-wwz9mh\" in resourcegroup=\"rg-np-skip-minor-wwz9mh-j759r47mqgx8\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"name\": \"af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:50.487713455Z\",\n \"endTime\": \"2026-08-25T19:36:50.727476187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-wwz9mh at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e6c0\u003e: \n failed to create HCP cluster np-skip-minor-wwz9mh: failed waiting for cluster=\"np-skip-minor-wwz9mh\" in resourcegroup=\"rg-np-skip-minor-wwz9mh-j759r47mqgx8\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"name\": \"af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:50.487713455Z\",\n \"endTime\": \"2026-08-25T19:36:50.727476187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"29398e5014283f2012caa0762efe0f3e00d3d7c1a1fb018ef8360e5e9d3a47ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"a5e9145697fe5d26a793f74527aa00e2ec11f8e0d29fd25f899226a0c8ca6786","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc5c0\u003e: \n failed to create HCP cluster np-downgrade-9wkwgd: failed waiting for cluster=\"np-downgrade-9wkwgd\" in resourcegroup=\"rg-np-downgrade-9wkwgd-c6w4l726nv9w\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\",\n \"name\": \"b4ede516-845b-4e65-988d-164f84fc3105\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.55170955Z\",\n \"endTime\": \"2026-08-25T19:36:50.700241881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc5c0\u003e: \n failed to create HCP cluster np-downgrade-9wkwgd: failed waiting for cluster=\"np-downgrade-9wkwgd\" in resourcegroup=\"rg-np-downgrade-9wkwgd-c6w4l726nv9w\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\",\n \"name\": \"b4ede516-845b-4e65-988d-164f84fc3105\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.55170955Z\",\n \"endTime\": \"2026-08-25T19:36:50.700241881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3ed9d1f0cb8d4475d63a66a115b16337f22f5758fca2f69de21aa961184708f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"fdfa7061d24f7ad3a467bcd48be893b74bb83304ac5cba05b613cd11d788255d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-bcd9ws9rmx5f/cluster-candidate-4-20-n8sshv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00167e700\u003e: \n failed to create HCP cluster cluster-candidate-4-20-n8sshv: failed waiting for cluster=\"cluster-candidate-4-20-n8sshv\" in resourcegroup=\"rg-candidate-4-20-bcd9ws9rmx5f\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62a883b7-6482-4600-ac5e-4dcb74227510\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62a883b7-6482-4600-ac5e-4dcb74227510\",\n \"name\": \"62a883b7-6482-4600-ac5e-4dcb74227510\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:48.42593636Z\",\n \"endTime\": \"2026-08-25T19:36:50.740405953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-bcd9ws9rmx5f/cluster-candidate-4-20-n8sshv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00167e700\u003e: \n failed to create HCP cluster cluster-candidate-4-20-n8sshv: failed waiting for cluster=\"cluster-candidate-4-20-n8sshv\" in resourcegroup=\"rg-candidate-4-20-bcd9ws9rmx5f\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62a883b7-6482-4600-ac5e-4dcb74227510\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62a883b7-6482-4600-ac5e-4dcb74227510\",\n \"name\": \"62a883b7-6482-4600-ac5e-4dcb74227510\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:48.42593636Z\",\n \"endTime\": \"2026-08-25T19:36:50.740405953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"46e39056073309f2a3152e7b5768803154d80e2a4695f384f9deef8ce632788f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"73aa3f2260ddb71b0e4372ee5ba731cfabeb239d1e551a06c5cc7e6c31570148","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001d5620\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nt76rfflxn6j\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"name\": \"01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.227475903Z\",\n \"endTime\": \"2026-08-25T19:36:50.710544904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001d5620\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nt76rfflxn6j\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"name\": \"01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.227475903Z\",\n \"endTime\": \"2026-08-25T19:36:50.710544904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"48413c7feeb59820261626a3f2a23058f8774565bd333f45c8dc22a3d0c4924e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"521ed306a26b9f9dacf6c47c753c84342bde38e73eb4c5ef23908a8ca2f86d45","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001368540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fdrnwvkc7cpp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59ec6c1d-ad5d-4874-bd8c-f072aa6e3be3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59ec6c1d-ad5d-4874-bd8c-f072aa6e3be3\",\n \"name\": \"59ec6c1d-ad5d-4874-bd8c-f072aa6e3be3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:48.422989688Z\",\n \"endTime\": \"2026-08-25T19:36:50.666605102Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001368540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fdrnwvkc7cpp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59ec6c1d-ad5d-4874-bd8c-f072aa6e3be3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/59ec6c1d-ad5d-4874-bd8c-f072aa6e3be3\",\n \"name\": \"59ec6c1d-ad5d-4874-bd8c-f072aa6e3be3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:48.422989688Z\",\n \"endTime\": \"2026-08-25T19:36:50.666605102Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5290e849a447d8ab6c9fca4212256a352029d02e1a64ec8dd5aab69d1ac75412","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"c1c3211636089d001497e52a1029d888d6af4966bdcf5a039a78f8eb9fff0edc","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ea320\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h5k4cp2j9wxp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"name\": \"36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:46.898858572Z\",\n \"endTime\": \"2026-08-25T19:36:50.642648068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ea320\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h5k4cp2j9wxp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"name\": \"36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:46.898858572Z\",\n \"endTime\": \"2026-08-25T19:36:50.642648068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5382e1a47c782de16a41e8be57415c79d067d5b630ea470782dc8c4c8055cc5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"8467082f94f1cddceb7436a951e2eb6e01b80042e0072d9c5e4e2e131f794e55","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-9htd5n\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001152240\u003e: \n failed to create HCP cluster np-noedge-9htd5n: failed waiting for cluster=\"np-noedge-9htd5n\" in resourcegroup=\"rg-np-noedge-9htd5n-qtggmt8fzz7r\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae2ad05f-8b7a-45e5-9e4a-29d98cd23cc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae2ad05f-8b7a-45e5-9e4a-29d98cd23cc5\",\n \"name\": \"ae2ad05f-8b7a-45e5-9e4a-29d98cd23cc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.591709095Z\",\n \"endTime\": \"2026-08-25T19:36:50.690482981Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-9htd5n\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001152240\u003e: \n failed to create HCP cluster np-noedge-9htd5n: failed waiting for cluster=\"np-noedge-9htd5n\" in resourcegroup=\"rg-np-noedge-9htd5n-qtggmt8fzz7r\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae2ad05f-8b7a-45e5-9e4a-29d98cd23cc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae2ad05f-8b7a-45e5-9e4a-29d98cd23cc5\",\n \"name\": \"ae2ad05f-8b7a-45e5-9e4a-29d98cd23cc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.591709095Z\",\n \"endTime\": \"2026-08-25T19:36:50.690482981Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70b3688b5c3d95536d9025a5393d7ea34229b95054fac5fddb7f433fb8c536fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"24ef7c09d5d3ddd04909d1a88daf40487983c7d0988d0358607e410a3fcbd326","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee8180\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rnmt78kw8wzg\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"name\": \"ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:28.849719343Z\",\n \"endTime\": \"2026-08-25T19:38:30.742891601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, azure-cloud-controller-manager, cluster-image-registry-operator, olm-operator, cluster-version-operator, ignition-server, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, catalog-operator, cluster-node-tuning-operator, ignition-server-proxy, ingress-operator, olm-collect-profiles, certified-operators-catalog, machine-approver, kube-controller-manager, dns-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-autoscaler, openshift-route-controller-manager, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee8180\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rnmt78kw8wzg\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"name\": \"ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:28.849719343Z\",\n \"endTime\": \"2026-08-25T19:38:30.742891601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, azure-cloud-controller-manager, cluster-image-registry-operator, olm-operator, cluster-version-operator, ignition-server, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, catalog-operator, cluster-node-tuning-operator, ignition-server-proxy, ingress-operator, olm-collect-profiles, certified-operators-catalog, machine-approver, kube-controller-manager, dns-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-autoscaler, openshift-route-controller-manager, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"78ccfd0ac38e4ed1899de6378067770c3131d5a221eaa1081fa676d0ebe7f9b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"4e3c6fa2936ba28e83ab9d88f6267d56844fa246b84839315cd5a1f94e2ab484","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e94fa0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-w7gfkxf8nw4v\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\",\n \"name\": \"8aa0882c-9021-414b-b83f-d1f980679679\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.25563444Z\",\n \"endTime\": \"2026-08-25T19:36:50.662196769Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, packageserver, dns-operator, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e94fa0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-w7gfkxf8nw4v\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\",\n \"name\": \"8aa0882c-9021-414b-b83f-d1f980679679\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.25563444Z\",\n \"endTime\": \"2026-08-25T19:36:50.662196769Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, packageserver, dns-operator, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"811e2d8cefb2bf34d1a6c8864131ac61ac6eeac254e1d03e0cc829f69597200b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"4ca0d90c5accc251f79a548ac4f235fa37708ec7a5a06deeb65a77bd99992647","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e2c60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fzg7bs5ws6ct\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"name\": \"444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:29.031939015Z\",\n \"endTime\": \"2026-08-25T19:36:30.68458199Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e2c60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fzg7bs5ws6ct\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"name\": \"444c3209-4e2e-4761-820a-0c2926e22f5c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:29.031939015Z\",\n \"endTime\": \"2026-08-25T19:36:30.68458199Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aac4d2b55915e86d753a384a5d375576eec1d2d2ba8264e59aa492de4fb022ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"e51e9dfee4ba4c9daa2b79ed992ce895a67497a0787c0bbfe0188313ac862ed6","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009d7b60\u003e: \n GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4971bff-3ccd-45d4-ae2d-00f32fedc469\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4971bff-3ccd-45d4-ae2d-00f32fedc469\",\n \"name\": \"d4971bff-3ccd-45d4-ae2d-00f32fedc469\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:11.040656627Z\",\n \"endTime\": \"2026-08-25T19:38:20.724383521Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009d7b60\u003e: \n GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4971bff-3ccd-45d4-ae2d-00f32fedc469\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4971bff-3ccd-45d4-ae2d-00f32fedc469\",\n \"name\": \"d4971bff-3ccd-45d4-ae2d-00f32fedc469\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:11.040656627Z\",\n \"endTime\": \"2026-08-25T19:38:20.724383521Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bbf9d9699e8acfb10e2eb692d527bc5c0d08954ad34fb6d53a5c630c8fb9939b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"8a342e0125bf2802c9ea767d7ffb9709438a1f4e72f2ad64f7e052e12c236331","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-fdqnm6\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e400\u003e: \n failed to create HCP cluster cluster-zstream-4-20-fdqnm6: failed waiting for cluster=\"cluster-zstream-4-20-fdqnm6\" in resourcegroup=\"rg-zstream-upgrade-4-20-jcxmnzcm2cwf\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c6bf604-afb6-4876-8c8c-842c103eb023\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c6bf604-afb6-4876-8c8c-842c103eb023\",\n \"name\": \"0c6bf604-afb6-4876-8c8c-842c103eb023\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:48.2405656Z\",\n \"endTime\": \"2026-08-25T19:36:50.736813407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-fdqnm6\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e400\u003e: \n failed to create HCP cluster cluster-zstream-4-20-fdqnm6: failed waiting for cluster=\"cluster-zstream-4-20-fdqnm6\" in resourcegroup=\"rg-zstream-upgrade-4-20-jcxmnzcm2cwf\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c6bf604-afb6-4876-8c8c-842c103eb023\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c6bf604-afb6-4876-8c8c-842c103eb023\",\n \"name\": \"0c6bf604-afb6-4876-8c8c-842c103eb023\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:48.2405656Z\",\n \"endTime\": \"2026-08-25T19:36:50.736813407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d75c57cebef9383b83b26917933b6235b20b13a04d5cfdc94150779bd0b303ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"f5fbd5480e218821391ad606bc369871f81ab79ca13513e6166733cd7e61e250","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4j5sc54kln8q/cluster-candidate-4-22-sf6h7k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e0360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-sf6h7k: failed waiting for cluster=\"cluster-candidate-4-22-sf6h7k\" in resourcegroup=\"rg-candidate-4-22-4j5sc54kln8q\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"name\": \"de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:37.268675469Z\",\n \"endTime\": \"2026-08-25T19:36:40.707957411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, dns-operator, catalog-operator, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4j5sc54kln8q/cluster-candidate-4-22-sf6h7k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e0360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-sf6h7k: failed waiting for cluster=\"cluster-candidate-4-22-sf6h7k\" in resourcegroup=\"rg-candidate-4-22-4j5sc54kln8q\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"name\": \"de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:37.268675469Z\",\n \"endTime\": \"2026-08-25T19:36:40.707957411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, dns-operator, catalog-operator, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d76ab51263577fa6b68a1a7e2e6fc6d175b930068580a27d3388cafdd9fb0c5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"5648256268d040d66790c83ce020dd81edbe55daec10ac344f210eb0a2221d9a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d98e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-lnqr4972dtpb\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"name\": \"f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:20.791598979Z\",\n \"endTime\": \"2026-08-25T19:37:20.738606915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d98e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-lnqr4972dtpb\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"name\": \"f003e8b0-f989-4c38-8deb-37f35273e17d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:20.791598979Z\",\n \"endTime\": \"2026-08-25T19:37:20.738606915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"172dca1d402dce2af7dedfe8fcbd440e4917e443080839ac33b06bea5d61d387","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2bbc4b2680c74c07c72fbfce871f0cb6156faa2c189fd54f8654078b1e98048","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"620efc4df137449edf6e36bc5ee396e617e4c93be1f20afe74e9e360cbb49723","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w5dw7vtzvxkx/cluster-hs-pre-dlbp6v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e760\u003e: \n failed to create HCP cluster cluster-hs-pre-dlbp6v: failed waiting for cluster=\"cluster-hs-pre-dlbp6v\" in resourcegroup=\"rg-hs-pre--w5dw7vtzvxkx\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"name\": \"f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:47.341416632Z\",\n \"endTime\": \"2026-08-25T19:36:50.700032947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w5dw7vtzvxkx/cluster-hs-pre-dlbp6v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e760\u003e: \n failed to create HCP cluster cluster-hs-pre-dlbp6v: failed waiting for cluster=\"cluster-hs-pre-dlbp6v\" in resourcegroup=\"rg-hs-pre--w5dw7vtzvxkx\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"name\": \"f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:47.341416632Z\",\n \"endTime\": \"2026-08-25T19:36:50.700032947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7ac84c870911ede55efafef711d64b94f6d96077275fe16906f8dd1476aa458","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","failed_at":"e2e","signature_id":"daeffcbe55429e419c5333569b8df49743acfca0ee5ac2e58304d2421599fcfb","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008921a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-xmcv5s5h772x\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc8d06df-511b-4bd2-8897-cd5d7c8479a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc8d06df-511b-4bd2-8897-cd5d7c8479a4\",\n \"name\": \"cc8d06df-511b-4bd2-8897-cd5d7c8479a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.745775666Z\",\n \"endTime\": \"2026-08-25T19:36:50.711273902Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008921a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-xmcv5s5h772x\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc8d06df-511b-4bd2-8897-cd5d7c8479a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc8d06df-511b-4bd2-8897-cd5d7c8479a4\",\n \"name\": \"cc8d06df-511b-4bd2-8897-cd5d7c8479a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:49.745775666Z\",\n \"endTime\": \"2026-08-25T19:36:50.711273902Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","172dca1d402dce2af7dedfe8fcbd440e4917e443080839ac33b06bea5d61d387","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6682,"pr_state":"open","pr_sha":"63b96f6864057d6e11490446bde9c2425aa70745","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:14:37Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3b5c49ca4e0ce86cfb9bfd4c23f14937b42a77702a657d417136e7ba70cc6905","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","failed_at":"e2e","signature_id":"365ceaf0468738d194ab343ea8045834af13aae002f5e53c5e41f490e916f1cc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vs25bgt56m2n/cluster-candidate-4-22-mbk6dn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e4e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-mbk6dn: failed waiting for cluster=\"cluster-candidate-4-22-mbk6dn\" in resourcegroup=\"rg-candidate-4-22-vs25bgt56m2n\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"name\": \"c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:41.789123755Z\",\n \"endTime\": \"2026-08-25T19:36:42.565693183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vs25bgt56m2n/cluster-candidate-4-22-mbk6dn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e4e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-mbk6dn: failed waiting for cluster=\"cluster-candidate-4-22-mbk6dn\" in resourcegroup=\"rg-candidate-4-22-vs25bgt56m2n\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"name\": \"c94972c0-8210-4d51-960d-183f4fd8e840\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:41.789123755Z\",\n \"endTime\": \"2026-08-25T19:36:42.565693183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b65aa0a2dcad8a5746a6042c9d72438127c4cd055167b43f182d744295b81ae4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","failed_at":"e2e","signature_id":"0076fbe063547b07110db9c853686695f6d9b45408e17004b5c20f9f45967910","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001af61c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-q5hktmktf7r4\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n \"name\": \"656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:31.574727642Z\",\n \"endTime\": \"2026-08-25T19:36:32.549404211Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001af61c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-q5hktmktf7r4\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n \"name\": \"656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:31.574727642Z\",\n \"endTime\": \"2026-08-25T19:36:32.549404211Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3f2429188d01f5a7dba5f14bc7e8758729053eeb0ef01d9ce593299ea86ea7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","failed_at":"e2e","signature_id":"77c50bc17b9d53d03c763973e2fb5dbf388cfdf6e7d1846e1e64d2e0d6d3a12d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cdmspj5xt8hg/cluster-candidate-4-20-t5rwd6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8c2c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t5rwd6: failed waiting for cluster=\"cluster-candidate-4-20-t5rwd6\" in resourcegroup=\"rg-candidate-4-20-cdmspj5xt8hg\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n \"name\": \"5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:20.683071275Z\",\n \"endTime\": \"2026-08-25T19:36:22.56632343Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cdmspj5xt8hg/cluster-candidate-4-20-t5rwd6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8c2c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t5rwd6: failed waiting for cluster=\"cluster-candidate-4-20-t5rwd6\" in resourcegroup=\"rg-candidate-4-20-cdmspj5xt8hg\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n \"name\": \"5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:17:20.683071275Z\",\n \"endTime\": \"2026-08-25T19:36:22.56632343Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:14:12Z"},"failed_at":["alert","e2e"],"failed_test_count":13,"occurrences":[{"row_id":"1b2565884d5d05dafec3bc574acc7dd79193460356ef36a991b542653be7eac6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"0f5a2599c879fd0d600773df180f9eb5e17770f7cbb1109f8e5936709c16bd9c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e46360\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjpc76rq228c\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\",\n \"name\": \"812b9f46-3042-442f-9212-72e785e2f21e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:26:56.464409042Z\",\n \"endTime\": \"2026-08-25T19:46:01.698082602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e46360\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjpc76rq228c\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\",\n \"name\": \"812b9f46-3042-442f-9212-72e785e2f21e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:26:56.464409042Z\",\n \"endTime\": \"2026-08-25T19:46:01.698082602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1e2afe07aa0f3a1a2f2d289177bb3ef22ee820ca16756246d834c171eb1401c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"alert","signature_id":"0ca8bc99c3c40c8ea160690445d9fbca4420866493398175f96a3e330d91ed47","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:50:28Z\n Ended: 2026-08-25T19:56:27Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7285632-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7285632-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"53a803cc67ff9f41d07da9ed4809755fff17d7446f302e7d0e110d75b7df8a73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"383b14a099f2ce5791796cf71b4050d10ad8f6db01c076f62c792d982c8b659a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef6ce0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tlsr2l96c7tj\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e2c8c82-48e2-4243-8503-87bf2548e54a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e2c8c82-48e2-4243-8503-87bf2548e54a\",\n \"name\": \"2e2c8c82-48e2-4243-8503-87bf2548e54a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:48.454775047Z\",\n \"endTime\": \"2026-08-25T19:39:53.93473257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef6ce0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tlsr2l96c7tj\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e2c8c82-48e2-4243-8503-87bf2548e54a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e2c8c82-48e2-4243-8503-87bf2548e54a\",\n \"name\": \"2e2c8c82-48e2-4243-8503-87bf2548e54a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:48.454775047Z\",\n \"endTime\": \"2026-08-25T19:39:53.93473257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b929e6f49e82ac70c27e16a3785003e1827af97d92f9f74e0067e4283f1a75b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"62dce0e08ee026ebe516bd675e439a69e85ef9b00198fc35594656b8245c2bc5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4xrqktzlmtt5/cluster-candidate-4-22-rc2wqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019065e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-rc2wqs: failed waiting for cluster=\"cluster-candidate-4-22-rc2wqs\" in resourcegroup=\"rg-candidate-4-22-4xrqktzlmtt5\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"name\": \"af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:41.204224191Z\",\n \"endTime\": \"2026-08-25T19:37:43.979682955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4xrqktzlmtt5/cluster-candidate-4-22-rc2wqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019065e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-rc2wqs: failed waiting for cluster=\"cluster-candidate-4-22-rc2wqs\" in resourcegroup=\"rg-candidate-4-22-4xrqktzlmtt5\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"name\": \"af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:41.204224191Z\",\n \"endTime\": \"2026-08-25T19:37:43.979682955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"65963d58371f4feaf1ead13b45a131ed3c4d34cd31f41683e65892b6e7918355","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61b6c30a45e97c7036d5f9b696ce83c0095caace60a57c18e9ea53b5f92f5eaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"42b392a614867f2ce7fd49a1175ca961a846df1640ce64649fc51da9a67204d7","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00104e420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r77s4bbr26ht\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"name\": \"8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:33.22593953Z\",\n \"endTime\": \"2026-08-25T19:38:33.945524152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00104e420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r77s4bbr26ht\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"name\": \"8c31bd00-7879-4578-bf29-c883b2f23367\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:19:33.22593953Z\",\n \"endTime\": \"2026-08-25T19:38:33.945524152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56aad0b4262c7cedf5722ba39358ba8c22af49e4bbcaade93736c9c50ce6eb1a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"69a49866fea6f529aa3634fcba22578b87470626168bc498053a2129e6a2b3d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"236662ab2cf12e0855b7e39f0def5a11fc29e4cd526454501035df011b642ee5","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dda660\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-p8zmpzpm6xjn\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f989241c-3d96-441a-b63b-a0585c1bcaae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f989241c-3d96-441a-b63b-a0585c1bcaae\",\n \"name\": \"f989241c-3d96-441a-b63b-a0585c1bcaae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:21:03.055406744Z\",\n \"endTime\": \"2026-08-25T19:40:04.014032995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dda660\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-p8zmpzpm6xjn\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f989241c-3d96-441a-b63b-a0585c1bcaae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f989241c-3d96-441a-b63b-a0585c1bcaae\",\n \"name\": \"f989241c-3d96-441a-b63b-a0585c1bcaae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:21:03.055406744Z\",\n \"endTime\": \"2026-08-25T19:40:04.014032995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ca219ecefb60170d06072ff2737d9b15427e9e86736cc773ebcd16bb9ce3887","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"58bf9ab1eb1b504ee63dc64568b36cd225e7db6ef55de533f9e062770f4ac7f7","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ebb040\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vscttbdw75nd\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\",\n \"name\": \"818a62a8-8d93-459d-8682-cfab29d0efcb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:19.815809316Z\",\n \"endTime\": \"2026-08-25T19:39:23.985097515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ebb040\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vscttbdw75nd\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\",\n \"name\": \"818a62a8-8d93-459d-8682-cfab29d0efcb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:19.815809316Z\",\n \"endTime\": \"2026-08-25T19:39:23.985097515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9c26d8fe0ad40d0718eb4ee9f94525a39d0c34c6e49d2658fd180cedf5763f8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"d9c26ef58d5fcf8c49cf5a4e45b95fc06f0f9d202df046f2ea975e15fe32ba90","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-s4hvvsklfcf9/cluster-candidate-5-0-gsjn7m should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e40540\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gsjn7m: failed waiting for cluster=\"cluster-candidate-5-0-gsjn7m\" in resourcegroup=\"rg-candidate-5-0-s4hvvsklfcf9\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e832dd3f-7a90-4215-9902-dd695bf53830\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e832dd3f-7a90-4215-9902-dd695bf53830\",\n \"name\": \"e832dd3f-7a90-4215-9902-dd695bf53830\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:58.764166146Z\",\n \"endTime\": \"2026-08-25T19:38:04.005472085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-s4hvvsklfcf9/cluster-candidate-5-0-gsjn7m should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e40540\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gsjn7m: failed waiting for cluster=\"cluster-candidate-5-0-gsjn7m\" in resourcegroup=\"rg-candidate-5-0-s4hvvsklfcf9\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e832dd3f-7a90-4215-9902-dd695bf53830\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e832dd3f-7a90-4215-9902-dd695bf53830\",\n \"name\": \"e832dd3f-7a90-4215-9902-dd695bf53830\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:58.764166146Z\",\n \"endTime\": \"2026-08-25T19:38:04.005472085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a47ec841d7ebe99f2e8c8fa700f6fdfb6ba8203f913ef5bd625553e693d87a6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"c1d27f93ce1a22b612a32738d89512a0f0eca7cf6197d0e5f557147d4b9ce477","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--bxjdj7s2bb72/cluster-hs-pre-x5q6nc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001014540\u003e: \n failed to create HCP cluster cluster-hs-pre-x5q6nc: failed waiting for cluster=\"cluster-hs-pre-x5q6nc\" in resourcegroup=\"rg-hs-pre--bxjdj7s2bb72\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3031cd41-7b22-47fc-84e2-ccf263091ef3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3031cd41-7b22-47fc-84e2-ccf263091ef3\",\n \"name\": \"3031cd41-7b22-47fc-84e2-ccf263091ef3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:47.849564088Z\",\n \"endTime\": \"2026-08-25T19:37:54.012692084Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--bxjdj7s2bb72/cluster-hs-pre-x5q6nc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001014540\u003e: \n failed to create HCP cluster cluster-hs-pre-x5q6nc: failed waiting for cluster=\"cluster-hs-pre-x5q6nc\" in resourcegroup=\"rg-hs-pre--bxjdj7s2bb72\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3031cd41-7b22-47fc-84e2-ccf263091ef3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3031cd41-7b22-47fc-84e2-ccf263091ef3\",\n \"name\": \"3031cd41-7b22-47fc-84e2-ccf263091ef3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:47.849564088Z\",\n \"endTime\": \"2026-08-25T19:37:54.012692084Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a58b7a397803b693b7ca0e4043f7eab3084e784262c53ce04f7ad8fdb08d4ee5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"e198c3b1d32fdcefe7fbd48785191cecaf277da264464a07176e87432c33c183","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"f72cda8e54add5e222bb916eb78bd6882ff8c549adafd27df57b6813b5fa6eee","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba1000\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-qxkbvwr99gjg\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0c05265-8fed-419d-aff1-dcdd08f92fb7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0c05265-8fed-419d-aff1-dcdd08f92fb7\",\n \"name\": \"c0c05265-8fed-419d-aff1-dcdd08f92fb7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:37.030270275Z\",\n \"endTime\": \"2026-08-25T19:39:44.008332311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba1000\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-qxkbvwr99gjg\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0c05265-8fed-419d-aff1-dcdd08f92fb7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0c05265-8fed-419d-aff1-dcdd08f92fb7\",\n \"name\": \"c0c05265-8fed-419d-aff1-dcdd08f92fb7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:20:37.030270275Z\",\n \"endTime\": \"2026-08-25T19:39:44.008332311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e485c81f34e55d5bc3c307b94a3852360547420ffd6576082881e07d9314eb2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"e2e","signature_id":"f731da949e2cbc194b5598178bee5be2ece9258202ff2948a41a7f722a5d15b6","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000549240\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-fsxff7rdsplm\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39f5eef7-6a31-4ba4-846f-bc334ff904d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39f5eef7-6a31-4ba4-846f-bc334ff904d3\",\n \"name\": \"39f5eef7-6a31-4ba4-846f-bc334ff904d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:52.467295506Z\",\n \"endTime\": \"2026-08-25T19:37:53.994993616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000549240\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-fsxff7rdsplm\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39f5eef7-6a31-4ba4-846f-bc334ff904d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39f5eef7-6a31-4ba4-846f-bc334ff904d3\",\n \"name\": \"39f5eef7-6a31-4ba4-846f-bc334ff904d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:18:52.467295506Z\",\n \"endTime\": \"2026-08-25T19:37:53.994993616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f709f62001ef1cbb684bdc0eb16b04683e4e9c417457eda1f1d6c7239c0a5285","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","failed_at":"alert","signature_id":"f1a9a05cfde18cca1700edae4de1bd0b0a9bac2292183bc359385aab761dec79","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s) (3 unknown, 3 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:28:42Z\n Ended: 2026-08-25T19:34:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7285632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpomt4g7ejhvupmg1pa2ftfip0tt7s-t7e9k3n5s3k8v7n\", pod=\"router-86757c4497-58vsh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdpomt4g7ejhvupmg1pa2ftfip0tt7s-t7e9k3n5s3k8v7n/router-86757c4497-58vsh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T19:31:45Z\n Ended: 2026-08-25T19:46:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7285632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpp91o4a51u8ud5pd0k878jmno613j-a9x6d8g5g6s4k3k\", pod=\"router-6f4cfc5587-882fr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdpp91o4a51u8ud5pd0k878jmno613j-a9x6d8g5g6s4k3k/router-6f4cfc5587-882fr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T19:31:45Z\n Ended: 2026-08-25T19:45:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7285632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpphhse66i17mrc0qiflkqsar87e1t-private-kas\", pod=\"router-7df9f4bb49-g7ndx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdpphhse66i17mrc0qiflkqsar87e1t-private-kas/router-7df9f4bb49-g7ndx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","56aad0b4262c7cedf5722ba39358ba8c22af49e4bbcaade93736c9c50ce6eb1a","65963d58371f4feaf1ead13b45a131ed3c4d34cd31f41683e65892b6e7918355","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314609523363840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6638,"pr_state":"open","pr_sha":"c403afe8d857fa8c76e2da31fc1c9d3952ca11b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:14:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3236391b60aedf2bba182f1abf84eac17ffc07c619d2337064b04d36a5abe685","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314609523363840","occurred_at":"2026-08-25T18:14:10Z","failed_at":"provision","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-25T18:56:45.688Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T18:56:46.086Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-25T18:56:46.474Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3363840-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07\ntime=2026-08-25T18:56:46.527Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3363840-mgmt-1/providers/Microsoft.Resources/deployments/74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07/operationStatuses/08584139226791753333\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"7eff23c12e4a99825ffd97ce671ba00e24716cc4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:12:12Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"b75ed4b2aa72fadcaeb00bd9e1888befc122d3fdb7967bd005de17449b4285b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","occurred_at":"2026-08-25T18:12:12Z","failed_at":"e2e","signature_id":"f9759b9996e08511f5b632db38111c4fa4b734c76990458a0d0586bc1dc45bbb","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000d6a348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-t87xzxghf54p\" to finish deleting: GET https://rp.j8620928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"name\": \"37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:09:21.722027891Z\",\n \"endTime\": \"2026-08-25T19:33:29.709053496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdpegmnfgfgem1607a6eukk8b4p953h still exists (deletion dispatched at 2026-08-25T19:09:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000d6a348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-t87xzxghf54p\" to finish deleting: GET https://rp.j8620928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"name\": \"37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T19:09:21.722027891Z\",\n \"endTime\": \"2026-08-25T19:33:29.709053496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdpegmnfgfgem1607a6eukk8b4p953h still exists (deletion dispatched at 2026-08-25T19:09:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f40f4ba6e055195a99ee8ad8e1e14df24385941e8c09ddcee7bb6689d4069187","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","occurred_at":"2026-08-25T18:12:12Z","failed_at":"e2e","signature_id":"6fabb5bf9236add92797cdcc42cf1ddf33539333a8c7650c981d9fd3ebea687d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e56140\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-update-nodes-hcp-cluster-chtjsv\" in resourcegroup=\"nodepool-update-nodes-sz99wmd6668j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/nodepool-update-nodes-sz99wmd6668j/providers/Microsoft.Resources/deployments/customer-infra-np-update-nodes-hcp-cluster-chtjsv/operationStatuses/08584139228279467438\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ConflictError\\\",\\r\\n \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e56140\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-update-nodes-hcp-cluster-chtjsv\" in resourcegroup=\"nodepool-update-nodes-sz99wmd6668j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/nodepool-update-nodes-sz99wmd6668j/providers/Microsoft.Resources/deployments/customer-infra-np-update-nodes-hcp-cluster-chtjsv/operationStatuses/08584139228279467438\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ConflictError\\\",\\r\\n \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConflictError; detail message A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"5b6c113c041a9ab13b00c2ccb04eed854d19e65c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:10:21Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"c0d61910fe5888bf088f9f5079fdf9b8c2057ada4dd5632cfca925d0882c7dfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","occurred_at":"2026-08-25T18:10:21Z","failed_at":"e2e","signature_id":"b81c2e95d57f9e709b06188bf80fa3ab504036a41d889fef881537e3c942592c","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:153]: failed to scale up node pool np-update-nodes from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0024223c0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-gld8nsfdc76t\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-gld8nsfdc76t, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:153]: failed to scale up node pool np-update-nodes from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0024223c0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-gld8nsfdc76t\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-gld8nsfdc76t, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded"}},{"row_id":"e3a5dbc12a4acdee0baf317f2e1e5c9a3aaf8c3db8f8ab5dc300a7c242cbb706","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","occurred_at":"2026-08-25T18:10:21Z","failed_at":"alert","signature_id":"42ec01f8964f22f9352e4ea60e7c8934ff52d4ff3ff5e12af66623897e7c4751","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T19:13:57Z\n Ended: 2026-08-25T19:41:56Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6140160-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6140160-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311337039302656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"open","pr_sha":"ccc0138e9d9cba8c23cc66260ecb2042998ab2a6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:01:04Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ea59a0ce719afe544b6710099bc5bd72eb61f01e40a5e331f2208b0160ef919a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311337039302656","occurred_at":"2026-08-25T18:01:04Z","failed_at":"e2e","signature_id":"970a4e186dca3e58eb5fc30dda68bc89be799ddbb9763794ffcc3be928d20bb8","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-dm2lv9wtz9pd\" to finish creating: GET https://rp.j9302656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\",\n \"name\": \"0f240228-e711-46ec-8078-6d29993f3656\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:54:14.698555112Z\",\n \"endTime\": \"2026-08-25T19:13:18.75896335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-dm2lv9wtz9pd\" to finish creating: GET https://rp.j9302656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\",\n \"name\": \"0f240228-e711-46ec-8078-6d29993f3656\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:54:14.698555112Z\",\n \"endTime\": \"2026-08-25T19:13:18.75896335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T18:00:08Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"292ff278329527aae8286a202b7ac2fd522fb029498ade6fe0d60d109a6c04a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","failed_at":"e2e","signature_id":"f7690a17f140753a646175aaf07b86c6525354b0ca8e94c3547454592f5e4408","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8e5a0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-s4m9gpmjnjkn\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d8c1ef2-0bb9-4056-b78f-c00010404578\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d8c1ef2-0bb9-4056-b78f-c00010404578\",\n \"name\": \"5d8c1ef2-0bb9-4056-b78f-c00010404578\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:32.643817495Z\",\n \"endTime\": \"2026-08-25T19:16:33.085736544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8e5a0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-s4m9gpmjnjkn\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d8c1ef2-0bb9-4056-b78f-c00010404578\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d8c1ef2-0bb9-4056-b78f-c00010404578\",\n \"name\": \"5d8c1ef2-0bb9-4056-b78f-c00010404578\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:32.643817495Z\",\n \"endTime\": \"2026-08-25T19:16:33.085736544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2f85ff54135586637bb19c0b44e3880d8a2f1be6276d2ec29a7146207ce3d3ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","failed_at":"e2e","signature_id":"fc8cdd9240844df1f37ddc400bdd2c5a950866210e7f77cce8d91f909ab5d0c9","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001562540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-t5q77t9nznpj\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba6d65cd-777b-4718-ad6f-f4416ae13858\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba6d65cd-777b-4718-ad6f-f4416ae13858\",\n \"name\": \"ba6d65cd-777b-4718-ad6f-f4416ae13858\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:21.772029384Z\",\n \"endTime\": \"2026-08-25T19:16:23.085277421Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001562540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-t5q77t9nznpj\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba6d65cd-777b-4718-ad6f-f4416ae13858\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba6d65cd-777b-4718-ad6f-f4416ae13858\",\n \"name\": \"ba6d65cd-777b-4718-ad6f-f4416ae13858\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:21.772029384Z\",\n \"endTime\": \"2026-08-25T19:16:23.085277421Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b9cd0871364c1fb85018cad70aea3bbb64cb938c91503db15f726cc3706037b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","failed_at":"e2e","signature_id":"09f8cbe70cbae7527497ee988bb4e3ca9bd72951deeb0cf5d80d73cf512c0032","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dde460\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4rmtpmn4hkq5\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8c913f3-9cc4-4379-a1e7-07fb33bc22d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8c913f3-9cc4-4379-a1e7-07fb33bc22d7\",\n \"name\": \"e8c913f3-9cc4-4379-a1e7-07fb33bc22d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:58:41.575567001Z\",\n \"endTime\": \"2026-08-25T19:17:43.086444554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dde460\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4rmtpmn4hkq5\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8c913f3-9cc4-4379-a1e7-07fb33bc22d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8c913f3-9cc4-4379-a1e7-07fb33bc22d7\",\n \"name\": \"e8c913f3-9cc4-4379-a1e7-07fb33bc22d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:58:41.575567001Z\",\n \"endTime\": \"2026-08-25T19:17:43.086444554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2180a7b769b58b70872bd81aaf71c7c411fe16b13db423ca0639299b1442d23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","failed_at":"e2e","signature_id":"08d84795c86bdd39c92b3ef7ed4171353ac5d7a95629fb6767e9d5e53dab0575","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00141cb00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-mh47z45hngdj\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd28938c-3ebd-4b76-94b7-9f92e718f37e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd28938c-3ebd-4b76-94b7-9f92e718f37e\",\n \"name\": \"bd28938c-3ebd-4b76-94b7-9f92e718f37e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:23.514305148Z\",\n \"endTime\": \"2026-08-25T19:16:23.093948104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00141cb00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-mh47z45hngdj\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd28938c-3ebd-4b76-94b7-9f92e718f37e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd28938c-3ebd-4b76-94b7-9f92e718f37e\",\n \"name\": \"bd28938c-3ebd-4b76-94b7-9f92e718f37e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:23.514305148Z\",\n \"endTime\": \"2026-08-25T19:16:23.093948104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eb138417cb90872f22efffb8d5a768382132c4ab18ee6b87688fcfab1e2a93d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","failed_at":"e2e","signature_id":"87d15fc3b3d51f2f525f669b15efa9810a7a54c78626d39624a2206eb8b40682","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pf7hbvppg5pm/cluster-candidate-4-20-42jsf9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e192e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-42jsf9: failed waiting for cluster=\"cluster-candidate-4-20-42jsf9\" in resourcegroup=\"rg-candidate-4-20-pf7hbvppg5pm\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbe12fb2-b5e4-45d1-bde4-07c02a203b34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbe12fb2-b5e4-45d1-bde4-07c02a203b34\",\n \"name\": \"cbe12fb2-b5e4-45d1-bde4-07c02a203b34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:22.069599707Z\",\n \"endTime\": \"2026-08-25T19:16:23.011359626Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pf7hbvppg5pm/cluster-candidate-4-20-42jsf9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e192e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-42jsf9: failed waiting for cluster=\"cluster-candidate-4-20-42jsf9\" in resourcegroup=\"rg-candidate-4-20-pf7hbvppg5pm\" to finish creating: GET https://rp.j8157056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbe12fb2-b5e4-45d1-bde4-07c02a203b34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbe12fb2-b5e4-45d1-bde4-07c02a203b34\",\n \"name\": \"cbe12fb2-b5e4-45d1-bde4-07c02a203b34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T18:57:22.069599707Z\",\n \"endTime\": \"2026-08-25T19:16:23.011359626Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092310827523641344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6554,"pr_state":"open","pr_sha":"4e67bc77de1a80644494c4c53d6416cbb8167300","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T17:59:09Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b8b318b65e5aa4e015f5031207bf00ae95460086b88d33f13b0860b92cb98fd1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092310827523641344","occurred_at":"2026-08-25T17:59:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092310329328406528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6657,"pr_state":"open","pr_sha":"52e11dc21d1bec7350e29d0c4470c4221bcb0bd7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T17:37:17Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4f1f73bb108cfd9a14082e861b6a6b79db0f68fed98f578365466705f9c2b7a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092310329328406528","occurred_at":"2026-08-25T17:37:17Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092303895547088896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6660,"pr_state":"open","pr_sha":"a660b6e55fc0b5bbac8bc93588a1797519a9c76f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T17:31:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"427763772c2cd234e542a70b64cad19c14d6c447","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T16:56:19Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"7329a7fde413b146af286fe05fc052f6be28499aa50da6b97e9b9f5ab2c5b636","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","failed_at":"e2e","signature_id":"0b8690595b1dfed6e36bd3e52f32e3625ed1e752008ebb3edcf9215308afc984","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00041c4e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bdpbrgzdzc8\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"name\": \"ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:41:06.665554524Z\",\n \"endTime\": \"2026-08-25T18:00:10.653150122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00041c4e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bdpbrgzdzc8\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"name\": \"ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:41:06.665554524Z\",\n \"endTime\": \"2026-08-25T18:00:10.653150122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"996fcc1a26cb4be6e76e2960594e4434a5f211c55d04efe72058c762a82786b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","failed_at":"alert","signature_id":"e5fda378ff57d5b1940c8266f347e7ee86ada79c73f8812c6767fe80cbdefd69","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (2 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T17:51:59Z\n Ended: 2026-08-25T18:18:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9832320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdob9oikt2alns01e8d0dpcp8roft0r-d6g2f1k7v9j7g7l\", pod=\"router-5d69c54566-gvkv8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdob9oikt2alns01e8d0dpcp8roft0r-d6g2f1k7v9j7g7l/router-5d69c54566-gvkv8 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T17:52:56Z\n Ended: 2026-08-25T18:15:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9832320-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdobf6c1r1ht9a0tq9k4goo675i5316-m4g9c3h1h9r9b2n\", pod=\"router-d7b75fd4b-bsdjf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdobf6c1r1ht9a0tq9k4goo675i5316-m4g9c3h1h9r9b2n/router-d7b75fd4b-bsdjf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e3f4aadcdc0a0cf614d165031892abfb6a72f8fc3dad717f2a653412b2c372cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","failed_at":"e2e","signature_id":"ca1da36a392a93e6f3a6490b4497374f23e1bdcba26c42be24847ece14ed3f61","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-nrmg7s9gn5qf/cluster-candidate-5-0-fh2kkx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009006a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fh2kkx: failed waiting for cluster=\"cluster-candidate-5-0-fh2kkx\" in resourcegroup=\"rg-candidate-5-0-nrmg7s9gn5qf\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ec762f1c-df9c-4468-b767-2fc6999a5c21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ec762f1c-df9c-4468-b767-2fc6999a5c21\",\n \"name\": \"ec762f1c-df9c-4468-b767-2fc6999a5c21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:40:55.279145717Z\",\n \"endTime\": \"2026-08-25T18:00:00.669636178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-nrmg7s9gn5qf/cluster-candidate-5-0-fh2kkx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009006a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fh2kkx: failed waiting for cluster=\"cluster-candidate-5-0-fh2kkx\" in resourcegroup=\"rg-candidate-5-0-nrmg7s9gn5qf\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ec762f1c-df9c-4468-b767-2fc6999a5c21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ec762f1c-df9c-4468-b767-2fc6999a5c21\",\n \"name\": \"ec762f1c-df9c-4468-b767-2fc6999a5c21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:40:55.279145717Z\",\n \"endTime\": \"2026-08-25T18:00:00.669636178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fcb119bedb132f510cf59d6e964344cad1ecef0436d032f4974fe33689e150ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","failed_at":"e2e","signature_id":"0669947174a86499d5670bc164ea47f8372430e063dda27921fe41db635f2d1a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-t2xtfqljcgg9/cluster-candidate-4-21-g6x6dm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8b620\u003e: \n failed to create HCP cluster cluster-candidate-4-21-g6x6dm: failed waiting for cluster=\"cluster-candidate-4-21-g6x6dm\" in resourcegroup=\"rg-candidate-4-21-t2xtfqljcgg9\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ea1fb1d-7d0e-4741-b43f-34275080dcae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ea1fb1d-7d0e-4741-b43f-34275080dcae\",\n \"name\": \"9ea1fb1d-7d0e-4741-b43f-34275080dcae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:41:15.401973458Z\",\n \"endTime\": \"2026-08-25T18:00:20.575131507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-t2xtfqljcgg9/cluster-candidate-4-21-g6x6dm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8b620\u003e: \n failed to create HCP cluster cluster-candidate-4-21-g6x6dm: failed waiting for cluster=\"cluster-candidate-4-21-g6x6dm\" in resourcegroup=\"rg-candidate-4-21-t2xtfqljcgg9\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ea1fb1d-7d0e-4741-b43f-34275080dcae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ea1fb1d-7d0e-4741-b43f-34275080dcae\",\n \"name\": \"9ea1fb1d-7d0e-4741-b43f-34275080dcae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:41:15.401973458Z\",\n \"endTime\": \"2026-08-25T18:00:20.575131507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5958,"pr_state":"open","pr_sha":"ea2dde0a91dc9ddee9fd9192846ece1b78645d33","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T16:12:48Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0e4f142d2520de8570357f2d6ad277cfa960d73aab6b8f82db99a74ac3f7b6b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","failed_at":"e2e","signature_id":"b168e3a1107e3e1902272b9ff4556d75fd8dfe44f624dfd2607bc5f72bb938f0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-sgkx2hv5dw5t/cluster-candidate-4-21-jn7kfp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b1c3a0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jn7kfp: failed waiting for cluster=\"cluster-candidate-4-21-jn7kfp\" in resourcegroup=\"rg-candidate-4-21-sgkx2hv5dw5t\" to finish creating: GET https://rp.j9793920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab3cc2a4-9ba0-4171-85d4-cdbb942a29a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab3cc2a4-9ba0-4171-85d4-cdbb942a29a4\",\n \"name\": \"ab3cc2a4-9ba0-4171-85d4-cdbb942a29a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:54:12.093832515Z\",\n \"endTime\": \"2026-08-25T17:13:15.538537349Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-sgkx2hv5dw5t/cluster-candidate-4-21-jn7kfp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b1c3a0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jn7kfp: failed waiting for cluster=\"cluster-candidate-4-21-jn7kfp\" in resourcegroup=\"rg-candidate-4-21-sgkx2hv5dw5t\" to finish creating: GET https://rp.j9793920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab3cc2a4-9ba0-4171-85d4-cdbb942a29a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab3cc2a4-9ba0-4171-85d4-cdbb942a29a4\",\n \"name\": \"ab3cc2a4-9ba0-4171-85d4-cdbb942a29a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:54:12.093832515Z\",\n \"endTime\": \"2026-08-25T17:13:15.538537349Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"245ddf12fc6bcd9effb32142808fcdb21a713b6694044e2bf4702b1e80ec0b19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","failed_at":"e2e","signature_id":"f71dae65ad1aafde8d4f6ae00fb97c69b8fec2ec7c262dff3ab35decf7d7de86","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--tfndrjk9sqsf/cluster-hs-pre-xzndj9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010886c0\u003e: \n failed to create HCP cluster cluster-hs-pre-xzndj9: failed waiting for cluster=\"cluster-hs-pre-xzndj9\" in resourcegroup=\"rg-hs-pre--tfndrjk9sqsf\" to finish creating: GET https://rp.j9793920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1822fb23-da51-4cef-a546-5dab91c6a66e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1822fb23-da51-4cef-a546-5dab91c6a66e\",\n \"name\": \"1822fb23-da51-4cef-a546-5dab91c6a66e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:54:12.394756997Z\",\n \"endTime\": \"2026-08-25T17:13:15.586924639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--tfndrjk9sqsf/cluster-hs-pre-xzndj9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010886c0\u003e: \n failed to create HCP cluster cluster-hs-pre-xzndj9: failed waiting for cluster=\"cluster-hs-pre-xzndj9\" in resourcegroup=\"rg-hs-pre--tfndrjk9sqsf\" to finish creating: GET https://rp.j9793920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1822fb23-da51-4cef-a546-5dab91c6a66e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1822fb23-da51-4cef-a546-5dab91c6a66e\",\n \"name\": \"1822fb23-da51-4cef-a546-5dab91c6a66e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:54:12.394756997Z\",\n \"endTime\": \"2026-08-25T17:13:15.586924639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3d74b4ade7dcd7c8309dea7a921177852646ed53baf8ceaf48a5f232dc5767df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","failed_at":"alert","signature_id":"2d9cf619745584716b1d36a78f973479ee6f6a9974e831e0e6595f6f7c09c34e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T17:06:23Z\n Ended: 2026-08-25T17:27:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9793920-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdnld7nfnceg8m66m97qmk82pp6t0ti-v9d8m6w5n0e5p1k\", pod=\"router-79c9f75898-m4z8p\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdnld7nfnceg8m66m97qmk82pp6t0ti-v9d8m6w5n0e5p1k/router-79c9f75898-m4z8p has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"7eff23c12e4a99825ffd97ce671ba00e24716cc4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T16:12:43Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"4b51a86e4e1025f5ed8780b24aef4cc18ca86fd13e65db645e3865fff30f0e43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","failed_at":"e2e","signature_id":"87fa0e9ed440b5075b8c246f9d4ac0dfb3a65d6379a21ec17eac51d7fe33d902","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001132120\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-99sdw8x8fbq7\" to finish creating: GET https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/577acfc9-e788-457f-b3ee-99ff00da118b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/577acfc9-e788-457f-b3ee-99ff00da118b\",\n \"name\": \"577acfc9-e788-457f-b3ee-99ff00da118b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:59:34.564176682Z\",\n \"endTime\": \"2026-08-25T17:18:37.881780961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001132120\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-99sdw8x8fbq7\" to finish creating: GET https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/577acfc9-e788-457f-b3ee-99ff00da118b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/577acfc9-e788-457f-b3ee-99ff00da118b\",\n \"name\": \"577acfc9-e788-457f-b3ee-99ff00da118b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:59:34.564176682Z\",\n \"endTime\": \"2026-08-25T17:18:37.881780961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d4ed26c5061e57e4e832f5d870863f6f173da5ad41c7ec0123784fcb0d8fa7a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","failed_at":"alert","signature_id":"84c9f5cec5ace865d6364245a39389a7b4680b6bfdf2958173dfcfd15754e303","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T17:12:13Z\n Ended: 2026-08-25T17:32:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8937216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdnntqml99r0h8mnakh8p60me0um74a-f3v4t8c7c6p2p7s\", pod=\"router-6b855cc9c6-fxfpr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdnntqml99r0h8mnakh8p60me0um74a-f3v4t8c7c6p2p7s/router-6b855cc9c6-fxfpr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e34a107054014ad9ad8b157749c5987a98187e6b2bc72309b7f1ce0c7325655f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","failed_at":"e2e","signature_id":"81a27532541d67e6137f3b18dfa24f47172a110e72475bc734463343f3f6e146","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000830340\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4d62fjrxvwg7\" to finish creating: GET https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1eaea7a-aae6-4966-8ffc-c5e041d39c1c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1eaea7a-aae6-4966-8ffc-c5e041d39c1c\",\n \"name\": \"d1eaea7a-aae6-4966-8ffc-c5e041d39c1c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:59:44.084689304Z\",\n \"endTime\": \"2026-08-25T17:18:47.865081649Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000830340\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4d62fjrxvwg7\" to finish creating: GET https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1eaea7a-aae6-4966-8ffc-c5e041d39c1c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1eaea7a-aae6-4966-8ffc-c5e041d39c1c\",\n \"name\": \"d1eaea7a-aae6-4966-8ffc-c5e041d39c1c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:59:44.084689304Z\",\n \"endTime\": \"2026-08-25T17:18:47.865081649Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284038717902848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T16:12:42Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4d1a17cb3bc690b8acd8e0c4001913965aa88176fa6b24acb5e1e6e542cbb36e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284038717902848","occurred_at":"2026-08-25T16:12:42Z","failed_at":"alert","signature_id":"7a5c0ec1b1bbd3d90f56d54a6d79e27df4b0c9b8be003edb69252609bc761f85","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T17:21:22Z\n Ended: 2026-08-25T17:27:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7902848-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdns5ree69mkiips5b97sq4nvu4khh3-y9s8x4j8f0u0h4h\", pod=\"router-7857668d8b-4lm5j\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdns5ree69mkiips5b97sq4nvu4khh3-y9s8x4j8f0u0h4h/router-7857668d8b-4lm5j has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092283360171790336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6630,"pr_state":"open","pr_sha":"53966db5bb00eb465ea4270e8fed068375360c6c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T16:10:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2cfb2d86eafba0896d442d9480d96c04bce7f755376cc5f73645ff648a57a758","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092283360171790336","occurred_at":"2026-08-25T16:10:00Z","failed_at":"e2e","signature_id":"e091e7c3963b1d439bcf214121c57fa63b1447fa3dcc926afb3e313210a13c5e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100b1a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-frm9xtk8fnxw\" to finish creating: GET https://rp.j1790336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fcd9358-9c18-42ba-8b4c-182393bd49da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fcd9358-9c18-42ba-8b4c-182393bd49da\",\n \"name\": \"8fcd9358-9c18-42ba-8b4c-182393bd49da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:07:37.992364096Z\",\n \"endTime\": \"2026-08-25T17:26:38.742090983Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100b1a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-frm9xtk8fnxw\" to finish creating: GET https://rp.j1790336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fcd9358-9c18-42ba-8b4c-182393bd49da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fcd9358-9c18-42ba-8b4c-182393bd49da\",\n \"name\": \"8fcd9358-9c18-42ba-8b4c-182393bd49da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T17:07:37.992364096Z\",\n \"endTime\": \"2026-08-25T17:26:38.742090983Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092280350733504512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6232,"pr_state":"open","pr_sha":"3e7f8503f0644e904aa5638ffccc85dcf305eb9c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T15:58:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6679,"pr_state":"open","pr_sha":"e33b5d3bf0fefbd8ec3be3d61d4b150c0c49acdc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T15:55:04Z"},"failed_at":["alert","e2e"],"failed_test_count":11,"occurrences":[{"row_id":"095fb0f89fcd4fa8b4c70d558d94d40fc07b892cfec6303ed7c70fa0a7b6a870","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"473e68c054364e4d1143001e092d3c5f19b78d56a2f0385244813ef7b9b36807","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9eb40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nmp69vm78h4k\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\",\n \"name\": \"584d4084-30a5-4233-a069-f2a0cf2b9043\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:03.375670894Z\",\n \"endTime\": \"2026-08-25T17:03:09.343723737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9eb40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nmp69vm78h4k\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\",\n \"name\": \"584d4084-30a5-4233-a069-f2a0cf2b9043\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:03.375670894Z\",\n \"endTime\": \"2026-08-25T17:03:09.343723737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2f45330175131dbd3246319f59456cdb62dba535d3b1982d3d5747f00eeb426a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"39a692c16b5bb74904ad081a46289002852a85390ea4850bc67236884f067be9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"aa50862b2be22c2ff41532ef6198e1092baabb54df9335776d5e11eaa6a79bd4","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e502a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nqflzjdjzf4p\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/293580a0-972a-4849-9fbf-bbd6db570984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/293580a0-972a-4849-9fbf-bbd6db570984\",\n \"name\": \"293580a0-972a-4849-9fbf-bbd6db570984\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:29.605638715Z\",\n \"endTime\": \"2026-08-25T17:01:29.31272361Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e502a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nqflzjdjzf4p\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/293580a0-972a-4849-9fbf-bbd6db570984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/293580a0-972a-4849-9fbf-bbd6db570984\",\n \"name\": \"293580a0-972a-4849-9fbf-bbd6db570984\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:29.605638715Z\",\n \"endTime\": \"2026-08-25T17:01:29.31272361Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"57d50e01642afe09a371d23cc249c8c5e38ccaa963c3ccd83623ca73483fbb89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"alert","signature_id":"fa55ad2520ac87c347911e5a9a91dd9cd00f969a783efd3c191a3eabc9b22503","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T16:56:51Z\n Ended: 2026-08-25T17:13:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0166528-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdnfuacmv6agt0dlstsphj72ajf634i-t5v3e1s2j1a3d2t\", pod=\"router-77cd575f5f-nmfnc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdnfuacmv6agt0dlstsphj72ajf634i-t5v3e1s2j1a3d2t/router-77cd575f5f-nmfnc has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"65aa09ae1c90ea9ef4ce313902a29b33e2ff40916e7b60c25754e74686074e76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"alert","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T17:40:35Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j0166528-svc\", component=\"backend\", name=\"triggercontrolplaneupgrade\", severity=\"warning\"\n Description: Backend controller workqueue triggercontrolplaneupgrade has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"6c717055db25ef339b06cd29f5784b58771491f427f49f04ad7f16c8930a8a20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"6d0a93471cca29b596f2665a2e4b14d8f14b38ab392d8cc47699b7365343a0d3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-xv5r4nm664qq/cluster-candidate-5-0-t7khkn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001240400\u003e: \n failed to create HCP cluster cluster-candidate-5-0-t7khkn: failed waiting for cluster=\"cluster-candidate-5-0-t7khkn\" in resourcegroup=\"rg-candidate-5-0-xv5r4nm664qq\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b985677c-5f0e-4eaf-9750-2cdf3f5d20fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b985677c-5f0e-4eaf-9750-2cdf3f5d20fd\",\n \"name\": \"b985677c-5f0e-4eaf-9750-2cdf3f5d20fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:32.948401484Z\",\n \"endTime\": \"2026-08-25T17:01:39.319371765Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-xv5r4nm664qq/cluster-candidate-5-0-t7khkn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001240400\u003e: \n failed to create HCP cluster cluster-candidate-5-0-t7khkn: failed waiting for cluster=\"cluster-candidate-5-0-t7khkn\" in resourcegroup=\"rg-candidate-5-0-xv5r4nm664qq\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b985677c-5f0e-4eaf-9750-2cdf3f5d20fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b985677c-5f0e-4eaf-9750-2cdf3f5d20fd\",\n \"name\": \"b985677c-5f0e-4eaf-9750-2cdf3f5d20fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:32.948401484Z\",\n \"endTime\": \"2026-08-25T17:01:39.319371765Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9be8c647c20680c5ee6acc25069245fe1c44b08ae9012f9630f21d0b1a0ce09d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"2d44f9e1ef87d22b647e496ad891393923d8b6071781ac4f3325409453b16aec","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881000\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-lwkzhddxfk2r\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d98a06a-da1b-47f1-8998-4d29a4c0c76a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d98a06a-da1b-47f1-8998-4d29a4c0c76a\",\n \"name\": \"8d98a06a-da1b-47f1-8998-4d29a4c0c76a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:47.284835124Z\",\n \"endTime\": \"2026-08-25T17:03:49.355549371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881000\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-lwkzhddxfk2r\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d98a06a-da1b-47f1-8998-4d29a4c0c76a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d98a06a-da1b-47f1-8998-4d29a4c0c76a\",\n \"name\": \"8d98a06a-da1b-47f1-8998-4d29a4c0c76a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:47.284835124Z\",\n \"endTime\": \"2026-08-25T17:03:49.355549371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a74e6dafe2b91a5f9a80d924c3dde1ea43632a3877147f054eb4669000a98909","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"3b0400f6feabfc5e3f875e41b2ad13cd2a30893f4a99364d4e6f721f7666dfca","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea0340\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-cm65c9f47lvs\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"name\": \"87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:45.179574291Z\",\n \"endTime\": \"2026-08-25T17:01:49.211726949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea0340\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-cm65c9f47lvs\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"name\": \"87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:45.179574291Z\",\n \"endTime\": \"2026-08-25T17:01:49.211726949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"94457f237fe77f556896b3c5dcbbf4f6fa7202d5ac7c0a13c7473270227156e1","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d42539b8a71438048831b2a704118d680557e1df4462b354a750d70f9554e8ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"431183966ffec9d316eabffd849e7f6e0b0f56934e37ad7b18ebee3f9d0c7785","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a542e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wdvkthrhsbcl\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7aecbab-9fd1-473f-9e9a-03ff3a146f3e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7aecbab-9fd1-473f-9e9a-03ff3a146f3e\",\n \"name\": \"a7aecbab-9fd1-473f-9e9a-03ff3a146f3e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:33.26817262Z\",\n \"endTime\": \"2026-08-25T17:01:39.343970634Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a542e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wdvkthrhsbcl\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7aecbab-9fd1-473f-9e9a-03ff3a146f3e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7aecbab-9fd1-473f-9e9a-03ff3a146f3e\",\n \"name\": \"a7aecbab-9fd1-473f-9e9a-03ff3a146f3e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:42:33.26817262Z\",\n \"endTime\": \"2026-08-25T17:01:39.343970634Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ea3124d64d4d672eec7ec166940068e111fce4b2f93b8d169f13d41a7e0d4fd1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"9022b451906b9a6a0432bfd8f6a4bba8e8dea2ae8057ca72799a51fd124f9bca","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:220]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000012048\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22, targetMinor=5.0) failed: clusterversion status.history has no version in target minor \"5.0\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:220]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000012048\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22, targetMinor=5.0) failed: clusterversion status.history has no version in target minor \"5.0\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"f12bfbcef778a14b45a27b34a21256b5ba46319fb226eb4d1c29d24e98e8d46b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","failed_at":"e2e","signature_id":"a34616cf01ee0e699438c4845004c7e68738e05288f74d735ab18aa5dbd40940","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00052b120\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4bcbzx62gx2r\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25c719fd-fce5-488f-a4c7-766d0744c777\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25c719fd-fce5-488f-a4c7-766d0744c777\",\n \"name\": \"25c719fd-fce5-488f-a4c7-766d0744c777\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:04.332076109Z\",\n \"endTime\": \"2026-08-25T17:03:09.229572599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00052b120\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4bcbzx62gx2r\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25c719fd-fce5-488f-a4c7-766d0744c777\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25c719fd-fce5-488f-a4c7-766d0744c777\",\n \"name\": \"25c719fd-fce5-488f-a4c7-766d0744c777\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:04.332076109Z\",\n \"endTime\": \"2026-08-25T17:03:09.229572599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","94457f237fe77f556896b3c5dcbbf4f6fa7202d5ac7c0a13c7473270227156e1","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6070,"pr_state":"open","pr_sha":"c1274d733bbc08c8eb6694a191f43bcddfe6ab8a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T15:23:59Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"285c845ad8c77bbd167e93baf88a151235518a145f926add63b9d2f626ee5a52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"alert","signature_id":"417c675e722a3d2e9207698265ffad01510d51821854d76bfab92157497f8c02","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T16:39:54Z\n Ended: 2026-08-25T17:06:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j0912128-svc\", component=\"backend\", name=\"clusterresourcescontroller\", severity=\"warning\"\n Description: Backend controller workqueue clusterresourcescontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"3ee16e5239a6ac0ef3b8d2800786be9c81d2906892b30a481bf9b847574b5c5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"44f45bd7c197771dae334f78215fd7f0e9a8f7b495a48ad1be4b5f40460c338c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00110af90\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00110af90\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","failure_pattern":"UpdateHCPCluster20260630 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded"}},{"row_id":"42dfc8ffbb3db3df71e4ebefcd65be3a769f538e1b7641f3c1049302d7e8ac3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"87e01991e35060c62797d52e4e26918b818f17bdf40e7387d9f553c0a433cb0e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e40258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-m7ds2xdkbpmr\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\",\n \"name\": \"424a05dd-5f96-45bf-a866-906148cef205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:34:49.049807041Z\",\n \"endTime\": \"2026-08-25T16:58:58.538368578Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lgco9mj0rdvfe15crftvs8408gm still exists (deletion dispatched at 2026-08-25T16:35:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e40258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-m7ds2xdkbpmr\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\",\n \"name\": \"424a05dd-5f96-45bf-a866-906148cef205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:34:49.049807041Z\",\n \"endTime\": \"2026-08-25T16:58:58.538368578Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lgco9mj0rdvfe15crftvs8408gm still exists (deletion dispatched at 2026-08-25T16:35:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"55cf21f3665512f37a057153bc5451fb07c6fa7a6bb29b53d642745f4cd679f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","failure_pattern":"expected 3 nodes to have original label key1=value1","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1"}},{"row_id":"8db70bfbd9e6d8b6cf9f904e4ffa4ed47a13bce7cedc76be1f9ddde7b29a9dab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"99c112613fb0611797f784fb621f4a92a4be1186e7f636e286eaba4372997bdb","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0033fc180\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0033fc180\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded"}},{"row_id":"96347cf82b3fec258de2f8b647eb8d142c4421a65db0406f3b775304cb756931","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"alert","signature_id":"e028319f9187bbe822ec5100ba5bf10f82963cc9e819d8d10a3fec4e07cba99c","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T17:23:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-d2656\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T17:23:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-d2656\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-08-25T17:29:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-08-25T17:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"b185f6ca6f5e76997a71a9a37677064a78806ad57b412137ea2c6fc86bca4ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"fcc9fdbef9bad418f67e51fbdb9dd4579ca43684aae6fa975e7587a95e591f82","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001a36438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-vkm2hsvvkr8h\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba3767be-4721-4bca-bfeb-3c6890864f09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba3767be-4721-4bca-bfeb-3c6890864f09\",\n \"name\": \"ba3767be-4721-4bca-bfeb-3c6890864f09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:24.055129576Z\",\n \"endTime\": \"2026-08-25T17:08:28.536915865Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6vvfklofaj57683jl810auik73me still exists (deletion dispatched at 2026-08-25T16:44:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001a36438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-vkm2hsvvkr8h\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba3767be-4721-4bca-bfeb-3c6890864f09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba3767be-4721-4bca-bfeb-3c6890864f09\",\n \"name\": \"ba3767be-4721-4bca-bfeb-3c6890864f09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:44:24.055129576Z\",\n \"endTime\": \"2026-08-25T17:08:28.536915865Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6vvfklofaj57683jl810auik73me still exists (deletion dispatched at 2026-08-25T16:44:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e96b0267c5056a165eff35d89b64673884293436d9d3a4d8af71b6788ef779d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"83f994f356f863bb37e48c00b2ef945191b4a6e47aa90ca4f068fad7c6f09ca9","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0019ac228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-lhs7stnz7jdx\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\",\n \"name\": \"ee794509-c973-4d3d-9aac-193a29f17172\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:40:54.657536996Z\",\n \"endTime\": \"2026-08-25T17:04:58.539200774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn70pm4stdfpd62ob77dhptdq67h82 still exists (deletion dispatched at 2026-08-25T16:40:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0019ac228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-lhs7stnz7jdx\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\",\n \"name\": \"ee794509-c973-4d3d-9aac-193a29f17172\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:40:54.657536996Z\",\n \"endTime\": \"2026-08-25T17:04:58.539200774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn70pm4stdfpd62ob77dhptdq67h82 still exists (deletion dispatched at 2026-08-25T16:40:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fba0630199b20fe33c83692929f4e015084b7256714e2477642aa421ba906587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","failed_at":"e2e","signature_id":"9c29cfd69e7610a41eb87b6317c7a73f3a7a8990c6ce0206c424b69b0ec00888","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001a3a078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vwd6cf59jxq2\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n \"name\": \"8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:34:33.14717886Z\",\n \"endTime\": \"2026-08-25T16:58:38.634142205Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lk5i59n2vdugq91i3qhl1hovg7u still exists (deletion dispatched at 2026-08-25T16:34:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001a3a078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vwd6cf59jxq2\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n \"name\": \"8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:34:33.14717886Z\",\n \"endTime\": \"2026-08-25T16:58:38.634142205Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lk5i59n2vdugq91i3qhl1hovg7u still exists (deletion dispatched at 2026-08-25T16:34:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6680/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092270173527478272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6680,"pr_state":"open","pr_sha":"46e74dc6c8cdc8f918e94bb0921d63ff767ffdcf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T15:17:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4566,"pr_state":"open","pr_sha":"9749b2d351ded24fec2bb28c4ddc42fbadee5a36","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T15:16:27Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0409ea33e696d22738bd2418f177148fa688770d48f93ee40545f4c65ede6b36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","failed_at":"e2e","signature_id":"a997df37747bd9ae4adc6c1d01f9efa7c9eeb6284eec6b1bc5d69624dc28ac10","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001350960\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-44brmcrqlwhd\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"name\": \"bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:55.65138247Z\",\n \"endTime\": \"2026-08-25T16:43:58.821172119Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001350960\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-44brmcrqlwhd\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"name\": \"bc250653-c8e3-42dd-bdb2-80628cccd921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:55.65138247Z\",\n \"endTime\": \"2026-08-25T16:43:58.821172119Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b3c5c6b82d824a461fdf0a9ceb99cba2cdf827bff592ead991249815b65b39a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"086175d3e1dd1bdba9a780e31513bade9053faf63229d9d70872f666f61854ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","failed_at":"e2e","signature_id":"320a0a462e18ca42da7b1b26ae6a21190926e34c2244492c5c4e0a1c81102a82","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-xhfxlf4rcz52/cluster-candidate-4-21-rvhxrv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f54d40\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rvhxrv: failed waiting for cluster=\"cluster-candidate-4-21-rvhxrv\" in resourcegroup=\"rg-candidate-4-21-xhfxlf4rcz52\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62063914-1f6c-4e07-b0d9-5145a898b73d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62063914-1f6c-4e07-b0d9-5145a898b73d\",\n \"name\": \"62063914-1f6c-4e07-b0d9-5145a898b73d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:49.237499205Z\",\n \"endTime\": \"2026-08-25T16:42:58.795937162Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-xhfxlf4rcz52/cluster-candidate-4-21-rvhxrv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f54d40\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rvhxrv: failed waiting for cluster=\"cluster-candidate-4-21-rvhxrv\" in resourcegroup=\"rg-candidate-4-21-xhfxlf4rcz52\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62063914-1f6c-4e07-b0d9-5145a898b73d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62063914-1f6c-4e07-b0d9-5145a898b73d\",\n \"name\": \"62063914-1f6c-4e07-b0d9-5145a898b73d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:49.237499205Z\",\n \"endTime\": \"2026-08-25T16:42:58.795937162Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"503eaee0896b94d689d5369f2c5c0c37544e8e6175d38964ca4aacdb14882f8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","failed_at":"e2e","signature_id":"4cde9318d023ca11a0bb6154c219327b20248fc8365ff58ea50734ae7b25de59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c4b48j\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001892580\u003e: \n failed to create HCP cluster cluster-zstream-4-21-c4b48j: failed waiting for cluster=\"cluster-zstream-4-21-c4b48j\" in resourcegroup=\"rg-zstream-upgrade-4-21-tjwbnczmk66w\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"name\": \"5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:22.258032156Z\",\n \"endTime\": \"2026-08-25T16:43:28.791772827Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c4b48j\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001892580\u003e: \n failed to create HCP cluster cluster-zstream-4-21-c4b48j: failed waiting for cluster=\"cluster-zstream-4-21-c4b48j\" in resourcegroup=\"rg-zstream-upgrade-4-21-tjwbnczmk66w\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"name\": \"5dd2277c-c818-44a4-91cb-db69efb21df4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:24:22.258032156Z\",\n \"endTime\": \"2026-08-25T16:43:28.791772827Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b737d1311207f88418b008ac26e9fb2e2e13895cec066d7642e1f2acb2ee783","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","failed_at":"e2e","signature_id":"f337aa7b62ef749e20f60dfec4dfcb7a4238a15c3da0837817bfff818d840f30","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138c280\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s8fx96sjztzk\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b66bae1c-c22e-47b7-adb2-c5dc89eaa631\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b66bae1c-c22e-47b7-adb2-c5dc89eaa631\",\n \"name\": \"b66bae1c-c22e-47b7-adb2-c5dc89eaa631\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:49.655363772Z\",\n \"endTime\": \"2026-08-25T16:42:58.742685687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138c280\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s8fx96sjztzk\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b66bae1c-c22e-47b7-adb2-c5dc89eaa631\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b66bae1c-c22e-47b7-adb2-c5dc89eaa631\",\n \"name\": \"b66bae1c-c22e-47b7-adb2-c5dc89eaa631\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:49.655363772Z\",\n \"endTime\": \"2026-08-25T16:42:58.742685687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aaa932617252d97a99f3b6d8d89b2b06d8177ae908f851c36d18eef1b3d16926","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","failed_at":"e2e","signature_id":"b94b017a26d7c85ad3a642438367f43e24dfb07c60186268412be86721621933","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--sgwjdjgmhg59/cluster-hs-pre-28tpcf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc840\u003e: \n failed to create HCP cluster cluster-hs-pre-28tpcf: failed waiting for cluster=\"cluster-hs-pre-28tpcf\" in resourcegroup=\"rg-hs-pre--sgwjdjgmhg59\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"name\": \"bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:47.826894918Z\",\n \"endTime\": \"2026-08-25T16:42:48.770504975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--sgwjdjgmhg59/cluster-hs-pre-28tpcf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fc840\u003e: \n failed to create HCP cluster cluster-hs-pre-28tpcf: failed waiting for cluster=\"cluster-hs-pre-28tpcf\" in resourcegroup=\"rg-hs-pre--sgwjdjgmhg59\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"name\": \"bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:47.826894918Z\",\n \"endTime\": \"2026-08-25T16:42:48.770504975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bd8944c9560147308a683885d199860a816c24f450fa26dcc2d571dd6c882239","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","4b3c5c6b82d824a461fdf0a9ceb99cba2cdf827bff592ead991249815b65b39a","6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","bd8944c9560147308a683885d199860a816c24f450fa26dcc2d571dd6c882239","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T15:16:24Z"},"failed_at":["alert","e2e"],"failed_test_count":21,"occurrences":[{"row_id":"0dfe82c81531f270703aa7f6e224790ccd255cd5ee5acbabef231c9d43422f54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"1ab36c45d2a3c6d8c0f25f766775b49a8fb0a98d92de9ba14bd6ca6e8846b72d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"8736192f6f4f645775b5c68f46d7ee7ddc0264e06e6677e85bab2a90cd28f2bc","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f71140\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7fhrl8nsqhmm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"name\": \"9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:28.727590226Z\",\n \"endTime\": \"2026-08-25T16:40:28.622162047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f71140\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7fhrl8nsqhmm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"name\": \"9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:28.727590226Z\",\n \"endTime\": \"2026-08-25T16:40:28.622162047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"39b2301f65c8b898af4dd6ae621c6e00fffc66640a334af169918614c4f5b2eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"f3706b7b68e9a79a6664d2f4e9bb40f34471b0eda04d386d956f5c3d42464cb6","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e62940\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-sd6rcg2rjkrp\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac0fd996-f807-4332-8f23-716715bbe1f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac0fd996-f807-4332-8f23-716715bbe1f9\",\n \"name\": \"ac0fd996-f807-4332-8f23-716715bbe1f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:28.235084078Z\",\n \"endTime\": \"2026-08-25T16:40:28.60703914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e62940\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-sd6rcg2rjkrp\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac0fd996-f807-4332-8f23-716715bbe1f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac0fd996-f807-4332-8f23-716715bbe1f9\",\n \"name\": \"ac0fd996-f807-4332-8f23-716715bbe1f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:28.235084078Z\",\n \"endTime\": \"2026-08-25T16:40:28.60703914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b6be637bdbf588a1347c6c02f0f541f069a560de57cf4e2b5e87cb9eb9f0312","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"338f3c10fa302ec66d489a7e252c6f877b54dac7f42009f33bf533ea79c5069d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-zgjbj5 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b1e160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zgjbj5: failed waiting for cluster=\"np-version-upgrade-cluster-zgjbj5\" in resourcegroup=\"rg-np-version-upgrade-zgjbj5-5lcrgtsl9x9p\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"name\": \"ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:59.293835394Z\",\n \"endTime\": \"2026-08-25T16:40:08.618237862Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-zgjbj5 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b1e160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zgjbj5: failed waiting for cluster=\"np-version-upgrade-cluster-zgjbj5\" in resourcegroup=\"rg-np-version-upgrade-zgjbj5-5lcrgtsl9x9p\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"name\": \"ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:59.293835394Z\",\n \"endTime\": \"2026-08-25T16:40:08.618237862Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2ffeafac71db9cc68f648812079b8ded624a6470e7c1704cbbbe007755392f55","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f9a1f9d025209071f75f4d263584722be72cfb605a726919bd4c0ddfa095899","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"9a5de514cbd74ec8858c418a7e3bc6208cf5c8a2a6c4d616b5ae583b533e5e79","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f54ca0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jm7j95xkmcf2\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fd76614-fbf5-4f0a-8889-fc415cb6c440\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fd76614-fbf5-4f0a-8889-fc415cb6c440\",\n \"name\": \"1fd76614-fbf5-4f0a-8889-fc415cb6c440\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:22:37.538637853Z\",\n \"endTime\": \"2026-08-25T16:41:38.64255925Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f54ca0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jm7j95xkmcf2\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fd76614-fbf5-4f0a-8889-fc415cb6c440\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fd76614-fbf5-4f0a-8889-fc415cb6c440\",\n \"name\": \"1fd76614-fbf5-4f0a-8889-fc415cb6c440\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:22:37.538637853Z\",\n \"endTime\": \"2026-08-25T16:41:38.64255925Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"487bc98050147b0b5d80b56c3477358cbaa116b754fe3fdd0e6586850506f204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"2501774b6fdfd7c419479e578ae84628be587408415c6fa7b689447828db6eaa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-fnczl99xh8kl/cluster-candidate-4-21-jscsmk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097d140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jscsmk: failed waiting for cluster=\"cluster-candidate-4-21-jscsmk\" in resourcegroup=\"rg-candidate-4-21-fnczl99xh8kl\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"name\": \"c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.440763508Z\",\n \"endTime\": \"2026-08-25T16:40:28.603771658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-fnczl99xh8kl/cluster-candidate-4-21-jscsmk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097d140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jscsmk: failed waiting for cluster=\"cluster-candidate-4-21-jscsmk\" in resourcegroup=\"rg-candidate-4-21-fnczl99xh8kl\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"name\": \"c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.440763508Z\",\n \"endTime\": \"2026-08-25T16:40:28.603771658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b04dc7590f9bf45d210ab8732f519c9a202ba140feec352b584759e5ca334f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"390df46832f7606456e52cdf4a307b68f19cb093f5c95774b04156c02106bca8","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018da440\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qd5fftk5xqnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"name\": \"3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:37.077702612Z\",\n \"endTime\": \"2026-08-25T16:40:38.597952289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018da440\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qd5fftk5xqnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"name\": \"3c61f442-7e7a-4287-a096-d40849d2499b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:37.077702612Z\",\n \"endTime\": \"2026-08-25T16:40:38.597952289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"247cc84147e4574f2635a831e66f7a46a1eb594df690c18944f1a846c19ad8ca","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, ingress-operator, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4e4bfc5d8e1b697ef14671036466a89c32d61a9b491f16f428dd7c82db3220d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"2e500e0b56bd09e2e3065c06d8cad98af6fc49618dd7b23cbffb2625133fb624","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee2ca0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bb6s68gx7kh6\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"name\": \"9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:40.989997281Z\",\n \"endTime\": \"2026-08-25T16:40:48.583774176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-policy-controller, packageserver, community-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee2ca0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bb6s68gx7kh6\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"name\": \"9561d49c-5877-4d79-855f-4df08a7e3fae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:40.989997281Z\",\n \"endTime\": \"2026-08-25T16:40:48.583774176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-policy-controller, packageserver, community-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"56812de0fbf360a9ff437177c992b6b4f2091e7c84ac7726e422572e1b8407b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"1f2d818f791aabf7ca966417dcc515df6d86223731a47e82cb83edd0e59ba4f7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-grhvwwgnmxfw/cluster-candidate-4-20-s6k958 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c140\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s6k958: failed waiting for cluster=\"cluster-candidate-4-20-s6k958\" in resourcegroup=\"rg-candidate-4-20-grhvwwgnmxfw\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"name\": \"d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:38.527859766Z\",\n \"endTime\": \"2026-08-25T16:40:38.616570204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server, ignition-server-proxy, certified-operators-catalog, catalog-operator, control-plane-pki-operator, kube-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, dns-operator, cluster-image-registry-operator, konnectivity-agent, redhat-operators-catalog, machine-approver, openshift-apiserver, openshift-controller-manager, ingress-operator, community-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-storage-operator, olm-operator, azure-cloud-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-grhvwwgnmxfw/cluster-candidate-4-20-s6k958 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c140\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s6k958: failed waiting for cluster=\"cluster-candidate-4-20-s6k958\" in resourcegroup=\"rg-candidate-4-20-grhvwwgnmxfw\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"name\": \"d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:38.527859766Z\",\n \"endTime\": \"2026-08-25T16:40:38.616570204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server, ignition-server-proxy, certified-operators-catalog, catalog-operator, control-plane-pki-operator, kube-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, dns-operator, cluster-image-registry-operator, konnectivity-agent, redhat-operators-catalog, machine-approver, openshift-apiserver, openshift-controller-manager, ingress-operator, community-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-storage-operator, olm-operator, azure-cloud-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60b3b8fd168e8e52f7b1a9427f26536d70083e3e7e5cc31845664c0e15f012cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"1e71ee8a9468b4add845994e78143a38f98dc804aebda495c2ae55adfc2902b9","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-prc4m6\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c9c3c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-prc4m6: failed waiting for cluster=\"cluster-zstream-4-20-prc4m6\" in resourcegroup=\"rg-zstream-upgrade-4-20-74gcx95r2n24\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"name\": \"7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:56.871797957Z\",\n \"endTime\": \"2026-08-25T16:39:58.557618251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-prc4m6\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c9c3c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-prc4m6: failed waiting for cluster=\"cluster-zstream-4-20-prc4m6\" in resourcegroup=\"rg-zstream-upgrade-4-20-74gcx95r2n24\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"name\": \"7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:20:56.871797957Z\",\n \"endTime\": \"2026-08-25T16:39:58.557618251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2ffeafac71db9cc68f648812079b8ded624a6470e7c1704cbbbe007755392f55","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6da845a96820283991ae66695f68ca5c0a8d918709ef428bd46955af80e3d9e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"7de4e86735efd5a13296026c22a5cfa492755634aa30ae6440c336d9757f0cba","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6nrgxcz8znbm/cluster-candidate-5-0-bz6rh7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001044860\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bz6rh7: failed waiting for cluster=\"cluster-candidate-5-0-bz6rh7\" in resourcegroup=\"rg-candidate-5-0-6nrgxcz8znbm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"name\": \"c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:17.696794179Z\",\n \"endTime\": \"2026-08-25T16:40:18.610816463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6nrgxcz8znbm/cluster-candidate-5-0-bz6rh7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001044860\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bz6rh7: failed waiting for cluster=\"cluster-candidate-5-0-bz6rh7\" in resourcegroup=\"rg-candidate-5-0-6nrgxcz8znbm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"name\": \"c0e826bb-86ad-484d-940a-a69d7c037731\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:17.696794179Z\",\n \"endTime\": \"2026-08-25T16:40:18.610816463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"74203b915f508fbe70f1b6d9158d496bff429130551e0c6eee83b6519273e581","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"460e7645261b38bfd4a8103a388c161c308276756d9b8ea62787b9b5009d2135","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff2780\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6hgvwqwzvj5t\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"name\": \"2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:39.227173208Z\",\n \"endTime\": \"2026-08-25T16:40:48.587074479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff2780\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6hgvwqwzvj5t\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"name\": \"2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:39.227173208Z\",\n \"endTime\": \"2026-08-25T16:40:48.587074479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8aa8dfecb65fbed5cf7114dca0becd99028c470a0f698ef234bd197a3285f784","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"alert","signature_id":"945dcc48e93f7e5479b3db04e970b41db897299b7ad75aca01031fd96f7fcf6e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s) (13 unknown, 11 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T16:37:02Z\n Ended: 2026-08-25T16:48:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn656b4osgukbth9n3lum5gp6jaj2n-j8n9u2x5g0l2u1m\", pod=\"router-5694df8766-77fjb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn656b4osgukbth9n3lum5gp6jaj2n-j8n9u2x5g0l2u1m/router-5694df8766-77fjb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T16:37:02Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn61l3u29crvag5ueaf2jktpso8sgs-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn61l3u29crvag5ueaf2jktpso8sgs-private-ingress/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T16:38:02Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn6h6baceee3dn4jbde3hiniipum3n-cilium-cluster\", pod=\"router-6b4c99498f-zsdlc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn6h6baceee3dn4jbde3hiniipum3n-cilium-cluster/router-6b4c99498f-zsdlc has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T16:38:02Z\n Ended: 2026-08-25T16:53:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn62e5gdi5s69o2fmo3td2ihh4399i-p8m2k0b4j9z1w9l\", pod=\"router-9cd49fbd4-5z2wr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn62e5gdi5s69o2fmo3td2ihh4399i-p8m2k0b4j9z1w9l/router-9cd49fbd4-5z2wr has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T16:38:02Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn6h6baceee3dn4jbde3hiniipum3n-cilium-cluster\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn6h6baceee3dn4jbde3hiniipum3n-cilium-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T16:38:02Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn656b4osgukbth9n3lum5gp6jaj2n-j8n9u2x5g0l2u1m\", pod=\"kube-apiserver-7fbbcc94d-fq9p6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn656b4osgukbth9n3lum5gp6jaj2n-j8n9u2x5g0l2u1m/kube-apiserver-7fbbcc94d-fq9p6 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T16:38:02Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn652p93biglu2ij3krnvkelsegk6n-x1n2h7r9m3o4t4m\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn652p93biglu2ij3krnvkelsegk6n-x1n2h7r9m3o4t4m/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T16:39:01Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn7153g2v0k31dooooj8u7k8mltn1a-agg-cluster\", pod=\"router-675c4686fd-2xgdc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn7153g2v0k31dooooj8u7k8mltn1a-agg-cluster/router-675c4686fd-2xgdc has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T16:39:01Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn62899n04b8kggs43s83ca16p2umu-oidc-wi-cluster\", pod=\"multus-admission-controller-856d98f646-pw2k6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn62899n04b8kggs43s83ca16p2umu-oidc-wi-cluster/multus-admission-controller-856d98f646-pw2k6 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T16:39:01Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn62899n04b8kggs43s83ca16p2umu-oidc-wi-cluster\", pod=\"network-node-identity-5486b5445d-g7cxx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn62899n04b8kggs43s83ca16p2umu-oidc-wi-cluster/network-node-identity-5486b5445d-g7cxx has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T16:39:01Z\n Ended: 2026-08-25T16:45:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn7153g2v0k31dooooj8u7k8mltn1a-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn7153g2v0k31dooooj8u7k8mltn1a-agg-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T16:39:01Z\n Ended: 2026-08-25T16:45:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn62899n04b8kggs43s83ca16p2umu-oidc-wi-cluster\", pod=\"ovnkube-control-plane-844f7dbfcd-25zdb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn62899n04b8kggs43s83ca16p2umu-oidc-wi-cluster/ovnkube-control-plane-844f7dbfcd-25zdb has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T16:40:01Z\n Ended: 2026-08-25T16:51:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0672512-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn5rat4libo1imashjsaa4uf5i00ah-i7z1w6u0o7q4z8r\", pod=\"router-7c687f7dd9-wlx8z\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn5rat4libo1imashjsaa4uf5i00ah-i7z1w6u0o7q4z8r/router-7c687f7dd9-wlx8z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"94410421e1548a2b1a97ca72ab6b343a9663a29f66b2967cc1780f6dfd5fe61a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"6d79c60b75c7247e2641f3408daf5532b9c4fe3a7b5290b37b8be3c077c8fcb6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000def640\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9f9z2nqvqrnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa85fcb-c56d-4b95-a957-349139c53706\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa85fcb-c56d-4b95-a957-349139c53706\",\n \"name\": \"8aa85fcb-c56d-4b95-a957-349139c53706\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:31.739649334Z\",\n \"endTime\": \"2026-08-25T16:42:38.476116853Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, kube-scheduler, openshift-route-controller-manager, packageserver, openshift-apiserver, dns-operator, ingress-operator, cluster-version-operator, cluster-network-operator, olm-operator, redhat-marketplace-catalog, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, openshift-controller-manager, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000def640\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9f9z2nqvqrnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa85fcb-c56d-4b95-a957-349139c53706\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa85fcb-c56d-4b95-a957-349139c53706\",\n \"name\": \"8aa85fcb-c56d-4b95-a957-349139c53706\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:23:31.739649334Z\",\n \"endTime\": \"2026-08-25T16:42:38.476116853Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, catalog-operator, redhat-operators-catalog, certified-operators-catalog, kube-scheduler, openshift-route-controller-manager, packageserver, openshift-apiserver, dns-operator, ingress-operator, cluster-version-operator, cluster-network-operator, olm-operator, redhat-marketplace-catalog, cluster-storage-operator, csi-snapshot-controller-operator, community-operators-catalog, openshift-controller-manager, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"97dcc0bf72354bbcef8d96716140f9bf45113ecb93cc261840796544ce0a25f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"4e02273c25987feb6c9bf4b5b1d1d5fd920f82e15dc281a89db33f4a9aed76a9","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002420\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-t2kb5kkgwfmt\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"name\": \"eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:26.515094904Z\",\n \"endTime\": \"2026-08-25T16:40:28.648310137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002420\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-t2kb5kkgwfmt\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"name\": \"eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:26.515094904Z\",\n \"endTime\": \"2026-08-25T16:40:28.648310137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a027e62d4049466bae1fbd1c0a8e7d1fda0d730295688c0f7574b7e8c612d489","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"alert","signature_id":"23aa9d7dee7c9e529060801e9167a47fb7a497985cb1d61e9db28b65884ea401","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T17:11:35Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j0672512-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0672512/providers/microsoft.monitor/accounts/services-j0672512\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"a49225d33554011cb1109f877d73fbb221ff195b515378b4dc03524cd075561d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"723d424b638d374276f7f3771ede71c77d1aeb31880363605a2c07414b4ed15d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000942ca0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kzttj9k8trpk\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"name\": \"848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:22:26.59789954Z\",\n \"endTime\": \"2026-08-25T16:41:28.524110603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ignition-server-proxy, cluster-policy-controller, hosted-cluster-config-operator, olm-collect-profiles, redhat-marketplace-catalog, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, openshift-apiserver, cluster-image-registry-operator, olm-operator, openshift-controller-manager, catalog-operator, community-operators-catalog, ignition-server, redhat-operators-catalog, konnectivity-agent, kube-scheduler, azure-cloud-controller-manager, certified-operators-catalog, machine-approver, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000942ca0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kzttj9k8trpk\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"name\": \"848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:22:26.59789954Z\",\n \"endTime\": \"2026-08-25T16:41:28.524110603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ignition-server-proxy, cluster-policy-controller, hosted-cluster-config-operator, olm-collect-profiles, redhat-marketplace-catalog, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, openshift-apiserver, cluster-image-registry-operator, olm-operator, openshift-controller-manager, catalog-operator, community-operators-catalog, ignition-server, redhat-operators-catalog, konnectivity-agent, kube-scheduler, azure-cloud-controller-manager, certified-operators-catalog, machine-approver, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a76b3a00b5bf48e029dac025887eb584ac6e80125749970fef02bd6e2b921e3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"77a4fccb98689590052e444032f6e0b2a81fc76d19cc8dedaba7314fbe9ef5b2","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--cgdbmlmgbcbn/cluster-hs-pre-5rbdg9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a340\u003e: \n failed to create HCP cluster cluster-hs-pre-5rbdg9: failed waiting for cluster=\"cluster-hs-pre-5rbdg9\" in resourcegroup=\"rg-hs-pre--cgdbmlmgbcbn\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb95384-7668-4364-9a3d-af25ebeda917\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb95384-7668-4364-9a3d-af25ebeda917\",\n \"name\": \"7cb95384-7668-4364-9a3d-af25ebeda917\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.709484961Z\",\n \"endTime\": \"2026-08-25T16:40:28.634231916Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--cgdbmlmgbcbn/cluster-hs-pre-5rbdg9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a340\u003e: \n failed to create HCP cluster cluster-hs-pre-5rbdg9: failed waiting for cluster=\"cluster-hs-pre-5rbdg9\" in resourcegroup=\"rg-hs-pre--cgdbmlmgbcbn\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb95384-7668-4364-9a3d-af25ebeda917\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb95384-7668-4364-9a3d-af25ebeda917\",\n \"name\": \"7cb95384-7668-4364-9a3d-af25ebeda917\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.709484961Z\",\n \"endTime\": \"2026-08-25T16:40:28.634231916Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b04505b1109319fa9e3768530a962d521086b558289d550b633008b2e2f663d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"148bb476ee1b7952c4359b6ada2341d8424d4da89438cb857f8b6bff27286eb4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-t55dzpdntqx7/cluster-candidate-4-22-4rzl9g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001738140\u003e: \n failed to create HCP cluster cluster-candidate-4-22-4rzl9g: failed waiting for cluster=\"cluster-candidate-4-22-4rzl9g\" in resourcegroup=\"rg-candidate-4-22-t55dzpdntqx7\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6edcffe4-fe02-4abf-a4aa-6cc45f718fbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6edcffe4-fe02-4abf-a4aa-6cc45f718fbb\",\n \"name\": \"6edcffe4-fe02-4abf-a4aa-6cc45f718fbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:37.209310875Z\",\n \"endTime\": \"2026-08-25T16:40:38.614625988Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-t55dzpdntqx7/cluster-candidate-4-22-4rzl9g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001738140\u003e: \n failed to create HCP cluster cluster-candidate-4-22-4rzl9g: failed waiting for cluster=\"cluster-candidate-4-22-4rzl9g\" in resourcegroup=\"rg-candidate-4-22-t55dzpdntqx7\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6edcffe4-fe02-4abf-a4aa-6cc45f718fbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6edcffe4-fe02-4abf-a4aa-6cc45f718fbb\",\n \"name\": \"6edcffe4-fe02-4abf-a4aa-6cc45f718fbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:37.209310875Z\",\n \"endTime\": \"2026-08-25T16:40:38.614625988Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cdd8c46b202d97392bef7641eeec07323b2f1871488df2f08a19180bb4a2fc61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"4f01337717137a4b91f0a8939b1c66d8d9f1d513029bbfdaa010a32c379be1fb","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010429c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-27m57z2w4jzb\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5176ecee-a4c1-4981-902c-1cba33665ee8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5176ecee-a4c1-4981-902c-1cba33665ee8\",\n \"name\": \"5176ecee-a4c1-4981-902c-1cba33665ee8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:00.853975604Z\",\n \"endTime\": \"2026-08-25T16:40:08.609691152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010429c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-27m57z2w4jzb\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5176ecee-a4c1-4981-902c-1cba33665ee8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5176ecee-a4c1-4981-902c-1cba33665ee8\",\n \"name\": \"5176ecee-a4c1-4981-902c-1cba33665ee8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:00.853975604Z\",\n \"endTime\": \"2026-08-25T16:40:08.609691152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fef1485ad06f88c67c6d3645a2aca90264771d038be5d1bce48d3a47e80f0c73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","failed_at":"e2e","signature_id":"04200eec52043650488776ea91f3a75dc9e4bb6cd15be64da7bc9e97ceff3c5e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-hrhcgp\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce540\u003e: \n failed to create HCP cluster cluster-zstream-5-0-hrhcgp: failed waiting for cluster=\"cluster-zstream-5-0-hrhcgp\" in resourcegroup=\"rg-zstream-upgrade-5-0-78mnn44hkdrc\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"name\": \"b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.785135797Z\",\n \"endTime\": \"2026-08-25T16:40:28.593240311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-hrhcgp\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dce540\u003e: \n failed to create HCP cluster cluster-zstream-5-0-hrhcgp: failed waiting for cluster=\"cluster-zstream-5-0-hrhcgp\" in resourcegroup=\"rg-zstream-upgrade-5-0-78mnn44hkdrc\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"name\": \"b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:21:27.785135797Z\",\n \"endTime\": \"2026-08-25T16:40:28.593240311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":21,"failure_pattern_ids":["1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","247cc84147e4574f2635a831e66f7a46a1eb594df690c18944f1a846c19ad8ca","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","2ffeafac71db9cc68f648812079b8ded624a6470e7c1704cbbbe007755392f55","4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":21,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"b89541a3f2c5a6ac4c3ef08a4057390880510f4d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T15:16:22Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"20057d79e4bfd9a1ed97f0b4719c28cfd49f412a183b76631b9339f5b5afefa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","failed_at":"alert","signature_id":"d5f85864d3d15c27d9d690ef128585ac4484e8c8af2bd1de02f0ebc7ecb160b9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T16:25:19Z\n Ended: 2026-08-25T16:33:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3036032-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn2a5elejp0cfku18d6rb3bv447o3c-private-ingress\", pod=\"router-5bcb798bf8-zh5bc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn2a5elejp0cfku18d6rb3bv447o3c-private-ingress/router-5bcb798bf8-zh5bc has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T16:25:19Z\n Ended: 2026-08-25T16:39:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3036032-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn2fvs43afb9nrsit8jlo6ct8g322g-v20260901\", pod=\"router-6f97cb49b9-8rhvp\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn2fvs43afb9nrsit8jlo6ct8g322g-v20260901/router-6f97cb49b9-8rhvp has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T16:25:19Z\n Ended: 2026-08-25T16:33:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3036032-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdn2dd3dlt36sr0dfi0c6siqogbnb8u-c4l2n5t0w7y1b4v\", pod=\"router-545548bbbf-bbdk6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdn2dd3dlt36sr0dfi0c6siqogbnb8u-c4l2n5t0w7y1b4v/router-545548bbbf-bbdk6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5a39e0b5311831065b33f7168f75e129c6efd468ab0dd6adeb1e37b90b29d7bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","failed_at":"e2e","signature_id":"87ca9ce38c81638d67c18762111fbc84a3f902586128fffc4d6a63e5d5d586d7","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001418640\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2x96b4gl9jc2\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e909f63-9fd3-40b3-89a2-647eb7c84e5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e909f63-9fd3-40b3-89a2-647eb7c84e5f\",\n \"name\": \"5e909f63-9fd3-40b3-89a2-647eb7c84e5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:14:50.211044644Z\",\n \"endTime\": \"2026-08-25T16:33:59.876273825Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001418640\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2x96b4gl9jc2\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e909f63-9fd3-40b3-89a2-647eb7c84e5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e909f63-9fd3-40b3-89a2-647eb7c84e5f\",\n \"name\": \"5e909f63-9fd3-40b3-89a2-647eb7c84e5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:14:50.211044644Z\",\n \"endTime\": \"2026-08-25T16:33:59.876273825Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9c52770f6dbbf1a6a8d2d5cf351859693a336fefac87034ed33ccefa19fe593","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","failed_at":"e2e","signature_id":"7f19b237d650c3e2c2dada758b0faf353ceddfaf46bd9c7e365fe5dd0616b720","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd01e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-kw5c7cgnwx6r\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8bc94cbd-7966-42ff-b2f3-8c3960b901a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8bc94cbd-7966-42ff-b2f3-8c3960b901a0\",\n \"name\": \"8bc94cbd-7966-42ff-b2f3-8c3960b901a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:14:34.084716184Z\",\n \"endTime\": \"2026-08-25T16:33:39.83687202Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd01e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-kw5c7cgnwx6r\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8bc94cbd-7966-42ff-b2f3-8c3960b901a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8bc94cbd-7966-42ff-b2f3-8c3960b901a0\",\n \"name\": \"8bc94cbd-7966-42ff-b2f3-8c3960b901a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:14:34.084716184Z\",\n \"endTime\": \"2026-08-25T16:33:39.83687202Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b313aeba609345e409b835506732d1b15ab7c2e954ee7396f5cfd61f8488dacf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","failed_at":"e2e","signature_id":"c301bcb0ad350b3e4edb18ab6b3112a4e915bcc1ccff4534d5bef4c12af07f76","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001200520\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-dzwjlx666575\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15fc6a54-c3a0-4f54-b9d1-9586d79644e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15fc6a54-c3a0-4f54-b9d1-9586d79644e9\",\n \"name\": \"15fc6a54-c3a0-4f54-b9d1-9586d79644e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:38.854250497Z\",\n \"endTime\": \"2026-08-25T16:32:39.953721503Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001200520\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-dzwjlx666575\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15fc6a54-c3a0-4f54-b9d1-9586d79644e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15fc6a54-c3a0-4f54-b9d1-9586d79644e9\",\n \"name\": \"15fc6a54-c3a0-4f54-b9d1-9586d79644e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:38.854250497Z\",\n \"endTime\": \"2026-08-25T16:32:39.953721503Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fd073ca4b07ca98db36d503e9a9055b369ce4001a44fbffcbbff3e1bdb3f8da5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","failed_at":"e2e","signature_id":"4bc11107f133f9177edbdf6e9f119a737521511f6edc5e691bc1878c9ba1f527","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86580\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qvdpwsdrhf5d\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"name\": \"11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:50.746895077Z\",\n \"endTime\": \"2026-08-25T16:32:59.846114727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86580\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qvdpwsdrhf5d\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"name\": \"11ae9dbb-7a03-4823-a826-d4717833e908\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:50.746895077Z\",\n \"endTime\": \"2026-08-25T16:32:59.846114727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fdcef1826fb24ef8f629bb5653e7ade971ee175b59babfd52ce8ec849c7c987d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","failed_at":"e2e","signature_id":"d90c9eedcebd8ebee811c56262835a367e830ea3ef985a6b9e0519703d7c4512","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f6760\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8qxvb7djrfqs\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\",\n \"name\": \"0ef3c075-0505-498f-acb7-bec851217620\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:28.807201061Z\",\n \"endTime\": \"2026-08-25T16:32:29.809496877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f6760\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8qxvb7djrfqs\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\",\n \"name\": \"0ef3c075-0505-498f-acb7-bec851217620\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T16:13:28.807201061Z\",\n \"endTime\": \"2026-08-25T16:32:29.809496877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092268396384096256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"open","pr_sha":"7a93dd5851f582b9fb50f30c5951fe2f48d805ce","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T15:10:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092262306963525632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6269,"pr_state":"closed","pr_sha":"0c4df626b4680a4230f24b939c285390ddbb3c77","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:46:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b543ca6901dc042f10bd8922db50e53f5d85083230902e3afeaa60a75588a276","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092262306963525632","occurred_at":"2026-08-25T14:46:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6676,"pr_state":"open","pr_sha":"5dfc8bf4922b8dfbdacf8f4e93f698eaa9dfd6ba","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:43:04Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1e9e6a6e98e162675ad9694a8cf9b2740460047397a772b7a243fef1b277fe30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","failed_at":"e2e","signature_id":"12cc269491c81e464e81232257f84d9ecbcb60190f84d4997ab10025e4d39380","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001342120\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wttjbjlspm75\" to finish creating: GET https://rp.j3905408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34bc5592-1de9-4751-8680-ffd928f8babf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34bc5592-1de9-4751-8680-ffd928f8babf\",\n \"name\": \"34bc5592-1de9-4751-8680-ffd928f8babf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:50:06.83845625Z\",\n \"endTime\": \"2026-08-25T16:09:10.978763078Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001342120\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wttjbjlspm75\" to finish creating: GET https://rp.j3905408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34bc5592-1de9-4751-8680-ffd928f8babf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34bc5592-1de9-4751-8680-ffd928f8babf\",\n \"name\": \"34bc5592-1de9-4751-8680-ffd928f8babf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:50:06.83845625Z\",\n \"endTime\": \"2026-08-25T16:09:10.978763078Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"63bda94c9e8a3a60c450cc79868ebeac90e2c33320ca6aa6772982fe48241b26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","failed_at":"alert","signature_id":"6619f8482211ef7b9931e9e6e5ce9917b4a89ba14b1e55a1266a1da1dc746d2a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s) (3 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T16:05:58Z\n Ended: 2026-08-25T16:14:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3905408-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdmn678cjt1bvv40dsep03g6dp7ja6g-k1r6c2q1w6q2r1l\", pod=\"router-697c8c9f66-6qpxz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdmn678cjt1bvv40dsep03g6dp7ja6g-k1r6c2q1w6q2r1l/router-697c8c9f66-6qpxz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T16:06:55Z\n Ended: 2026-08-25T16:13:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3905408-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdmnbtoke5te2abh8bef6jgas0h9teb-b2f9y5p2w9v8y9l\", pod=\"router-5489fff6b-q2fpx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdmnbtoke5te2abh8bef6jgas0h9teb-b2f9y5p2w9v8y9l/router-5489fff6b-q2fpx has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T16:06:55Z\n Ended: 2026-08-25T16:13:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3905408-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdmn8ppcpk07jkgi2itsdeesfjgkaq1-private-ingress\", pod=\"router-78b97948b9-g9t8j\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdmn8ppcpk07jkgi2itsdeesfjgkaq1-private-ingress/router-78b97948b9-g9t8j has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c49b1f0eb9273f17b0410a5893427a74a8afb68576c6f8ae45978f0ae9b2f72e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","failed_at":"e2e","signature_id":"8570bc7030e47ad260a5bdf4a8d1e6ec729b76caed85d2d85acd384bcef9aaf5","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--f9fqqjtpqkj4/cluster-hs-pre-b68c79 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02140\u003e: \n failed to create HCP cluster cluster-hs-pre-b68c79: failed waiting for cluster=\"cluster-hs-pre-b68c79\" in resourcegroup=\"rg-hs-pre--f9fqqjtpqkj4\" to finish creating: GET https://rp.j3905408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c87a3dc5-3324-4e8e-8ece-0c46e4236baf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c87a3dc5-3324-4e8e-8ece-0c46e4236baf\",\n \"name\": \"c87a3dc5-3324-4e8e-8ece-0c46e4236baf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:49:44.606597271Z\",\n \"endTime\": \"2026-08-25T16:08:50.991532908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--f9fqqjtpqkj4/cluster-hs-pre-b68c79 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02140\u003e: \n failed to create HCP cluster cluster-hs-pre-b68c79: failed waiting for cluster=\"cluster-hs-pre-b68c79\" in resourcegroup=\"rg-hs-pre--f9fqqjtpqkj4\" to finish creating: GET https://rp.j3905408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c87a3dc5-3324-4e8e-8ece-0c46e4236baf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c87a3dc5-3324-4e8e-8ece-0c46e4236baf\",\n \"name\": \"c87a3dc5-3324-4e8e-8ece-0c46e4236baf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:49:44.606597271Z\",\n \"endTime\": \"2026-08-25T16:08:50.991532908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e34e3a79dbb1ccb230f939d563d6a47e9cc164c3e5927f8b2cb1ccd0786ff42f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","failed_at":"e2e","signature_id":"adc58fbe078ffd0b9a7c1502fc926ccd76f020ceef6eb743ef3c165775530e5a","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffe300\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-v8qzqqn7phmz\" to finish creating: GET https://rp.j3905408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92b9a3b5-8aba-4a43-86bf-ab3ad8b4a9e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92b9a3b5-8aba-4a43-86bf-ab3ad8b4a9e9\",\n \"name\": \"92b9a3b5-8aba-4a43-86bf-ab3ad8b4a9e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:49:54.637795969Z\",\n \"endTime\": \"2026-08-25T16:09:01.068450014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffe300\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-v8qzqqn7phmz\" to finish creating: GET https://rp.j3905408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92b9a3b5-8aba-4a43-86bf-ab3ad8b4a9e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92b9a3b5-8aba-4a43-86bf-ab3ad8b4a9e9\",\n \"name\": \"92b9a3b5-8aba-4a43-86bf-ab3ad8b4a9e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:49:54.637795969Z\",\n \"endTime\": \"2026-08-25T16:09:01.068450014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6614,"pr_state":"open","pr_sha":"e0ff3c4234edb21f8dea0459288bc3760e1badc8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:42:10Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"036190a987369e6252a6b8862debb891de8867c2cff116167fd74be88ace2f08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","occurred_at":"2026-08-25T14:42:10Z","failed_at":"alert","signature_id":"0766b6645c6206a3fc1a103b5ed7d9fe07fd1625b37249763edfe4af9b817fbb","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:54:11Z\n Ended: 2026-08-25T16:01:15Z\n Severity: Sev3\n Description: Cosmos DB returned more than 100 throttled requests (HTTP 429 - request rate too large) summed over a 5-minute window, evaluated every minute. Sustained 429s mean the workload is exceeding provisioned throughput - investigate hot partitions or increase provisioned throughput (RU/s). A low, steady rate of 429s that client SDKs retry transparently can be normal. https://learn.microsoft.com/azure/cosmos-db/troubleshoot-request-rate-too-large","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0e72a5ca42c3a2d32bb83d02da4302579e640eee948d64f58a4f543c892be49b","failure_pattern":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 fired","search_query":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 fired"}},{"row_id":"a807c364c93747b59dc78d9f377ff6ba2cfb1dbf1b94240b383b36719b300f90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","occurred_at":"2026-08-25T14:42:10Z","failed_at":"alert","signature_id":"a433ce52553f210f473828de792bdb37ba8e12f31a35d7427eb780d885009233","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:53:01Z\n Ended: 2026-08-25T16:26:02Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7405952-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7405952-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0e72a5ca42c3a2d32bb83d02da4302579e640eee948d64f58a4f543c892be49b","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261105932963840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"5fdef9ddfd1ced2b8d1ba4a721a7a99e261c25b7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:41:34Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"78c00423dfd6f901beeca17299215f3036a973bb3b1d67a5039811d268fdf732","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261105932963840","occurred_at":"2026-08-25T14:41:34Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6629,"pr_state":"open","pr_sha":"c6d46781a7abcc4c551cdf9edbb601d3aee0cc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:41:22Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"0838a14221055dbce895e6c622a6e6cb92fc47a2be63554d1dd0d67010a233ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"alert","signature_id":"85470de5379b7a58f4475a5818e36e604131a03a4654ffc58a012b2817701ddb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:55:27Z\n Ended: 2026-08-25T16:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8947456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdmcm4cjq1180vpa50bh0k8c7f1ruqb-private-kas\", pod=\"router-787b99fdd9-wvbbf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdmcm4cjq1180vpa50bh0k8c7f1ruqb-private-kas/router-787b99fdd9-wvbbf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"15dc47c4d87b2ab51b9991aa3604af4d5be399474a38dfd7f356bc11986d42d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"fe4f032da53c9249c0554d3122ef5a19cf72dc9f02c7bcd2d72076136ebe7044","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pg2742k6drr6/cluster-candidate-4-20-s9697v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121c780\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s9697v: failed waiting for cluster=\"cluster-candidate-4-20-s9697v\" in resourcegroup=\"rg-candidate-4-20-pg2742k6drr6\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"name\": \"18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:38.293191515Z\",\n \"endTime\": \"2026-08-25T15:45:40.878934392Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pg2742k6drr6/cluster-candidate-4-20-s9697v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121c780\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s9697v: failed waiting for cluster=\"cluster-candidate-4-20-s9697v\" in resourcegroup=\"rg-candidate-4-20-pg2742k6drr6\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"name\": \"18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:38.293191515Z\",\n \"endTime\": \"2026-08-25T15:45:40.878934392Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"24acb41513c2840f05ba1f26a459ad850a12f5a86ddf099d8cb3607f64188d4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"alert","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:54:58Z\n Ended: 2026-08-25T16:00:59Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"70db2d61c2cd057fcd1ea1e913f1f94a57b2c2f5dbd99a0ba89a7c674d643f54","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 fired"}},{"row_id":"55dee5d7f854716fe0c4e4b070eb052f5d8f6fd7a58d30dcab17be41cfaedfe2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"a0a72a832973622295dca8311edc8acdd0d1be0a93774aa5a3c2c41f0a225677","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b87360\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-75tdl88p84mj\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70638949-645c-4e6e-baf8-3c094f5f0cd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70638949-645c-4e6e-baf8-3c094f5f0cd7\",\n \"name\": \"70638949-645c-4e6e-baf8-3c094f5f0cd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:29:01.756170917Z\",\n \"endTime\": \"2026-08-25T15:48:10.941060109Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b87360\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-75tdl88p84mj\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70638949-645c-4e6e-baf8-3c094f5f0cd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70638949-645c-4e6e-baf8-3c094f5f0cd7\",\n \"name\": \"70638949-645c-4e6e-baf8-3c094f5f0cd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:29:01.756170917Z\",\n \"endTime\": \"2026-08-25T15:48:10.941060109Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6092b356eae8df536eff84f240c5687c128643b2a3d47fecf1d4c6eea086c2bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"7afa6a3cb0a5460454f74e2a2b1999dd33e3f60965567464435dde991505b283","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--fgqd2jbq4t7r/cluster-hs-pre-5w9whg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00149e980\u003e: \n failed to create HCP cluster cluster-hs-pre-5w9whg: failed waiting for cluster=\"cluster-hs-pre-5w9whg\" in resourcegroup=\"rg-hs-pre--fgqd2jbq4t7r\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4456e8-aa9d-49fc-839c-5bca4f61aad1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4456e8-aa9d-49fc-839c-5bca4f61aad1\",\n \"name\": \"1d4456e8-aa9d-49fc-839c-5bca4f61aad1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:36.931048124Z\",\n \"endTime\": \"2026-08-25T15:45:40.904564768Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--fgqd2jbq4t7r/cluster-hs-pre-5w9whg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00149e980\u003e: \n failed to create HCP cluster cluster-hs-pre-5w9whg: failed waiting for cluster=\"cluster-hs-pre-5w9whg\" in resourcegroup=\"rg-hs-pre--fgqd2jbq4t7r\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4456e8-aa9d-49fc-839c-5bca4f61aad1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4456e8-aa9d-49fc-839c-5bca4f61aad1\",\n \"name\": \"1d4456e8-aa9d-49fc-839c-5bca4f61aad1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:36.931048124Z\",\n \"endTime\": \"2026-08-25T15:45:40.904564768Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6841b0fd57f0b027e74c05070b00eb23d329c5970ee1d8c86de61ccdfc4cc2fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"2326e62e81a715665c83cf8208b5b992003fcc3c884e154ce68d5a2525a2a304","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088e20\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-4qx7mnmm2gkt\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a3436a18-cf8e-49ed-b7cb-d8646cec6de4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a3436a18-cf8e-49ed-b7cb-d8646cec6de4\",\n \"name\": \"a3436a18-cf8e-49ed-b7cb-d8646cec6de4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:37.927050597Z\",\n \"endTime\": \"2026-08-25T15:45:40.902757825Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088e20\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-4qx7mnmm2gkt\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a3436a18-cf8e-49ed-b7cb-d8646cec6de4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a3436a18-cf8e-49ed-b7cb-d8646cec6de4\",\n \"name\": \"a3436a18-cf8e-49ed-b7cb-d8646cec6de4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:37.927050597Z\",\n \"endTime\": \"2026-08-25T15:45:40.902757825Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a1ef81e44a5227bb46c9d2270e391408a0ce6f657815226ec4a2c9d0357aadcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"b5155d1abd3afd29cd7d2bb03f43d6d1523d798062871307fb209c787f593dc6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d8480\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gxzkjn22dzjv\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b524ffb-dbcb-4815-9127-6b0aaa409edf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b524ffb-dbcb-4815-9127-6b0aaa409edf\",\n \"name\": \"0b524ffb-dbcb-4815-9127-6b0aaa409edf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:27:57.646849157Z\",\n \"endTime\": \"2026-08-25T15:47:00.925521895Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d8480\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gxzkjn22dzjv\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b524ffb-dbcb-4815-9127-6b0aaa409edf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b524ffb-dbcb-4815-9127-6b0aaa409edf\",\n \"name\": \"0b524ffb-dbcb-4815-9127-6b0aaa409edf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:27:57.646849157Z\",\n \"endTime\": \"2026-08-25T15:47:00.925521895Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb37fa034e218bdc2dac3f0bfffb38b40cc0bc651c6bbc93a37e086b1234fea4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"ca98a15fd3dbeb3c9c2ada441d28414afe433e197fe77478335816c4c14eb15a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070c080\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-6vr9n8kcsq5h\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4bad3fdc-14be-4d4e-8b59-9f0e4013950a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4bad3fdc-14be-4d4e-8b59-9f0e4013950a\",\n \"name\": \"4bad3fdc-14be-4d4e-8b59-9f0e4013950a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:27:19.498154616Z\",\n \"endTime\": \"2026-08-25T15:46:20.843517231Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070c080\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-6vr9n8kcsq5h\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4bad3fdc-14be-4d4e-8b59-9f0e4013950a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4bad3fdc-14be-4d4e-8b59-9f0e4013950a\",\n \"name\": \"4bad3fdc-14be-4d4e-8b59-9f0e4013950a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:27:19.498154616Z\",\n \"endTime\": \"2026-08-25T15:46:20.843517231Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6d8c0fae0caf1e463b897337e793e414a051a1752b11242df55613759db5ab0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","failed_at":"e2e","signature_id":"bd2bd12402128b3e6c7b2c7dc8891fd4bb2cbbd5fb1f6cc30af3b1c2b3e2de09","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111e6a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-kmrf4dpx7tg9\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56f96552-e780-4e7b-a16e-c37885f1c49d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56f96552-e780-4e7b-a16e-c37885f1c49d\",\n \"name\": \"56f96552-e780-4e7b-a16e-c37885f1c49d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:27:29.975055079Z\",\n \"endTime\": \"2026-08-25T15:46:30.914374064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111e6a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-kmrf4dpx7tg9\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56f96552-e780-4e7b-a16e-c37885f1c49d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56f96552-e780-4e7b-a16e-c37885f1c49d\",\n \"name\": \"56f96552-e780-4e7b-a16e-c37885f1c49d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:27:29.975055079Z\",\n \"endTime\": \"2026-08-25T15:46:30.914374064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","70db2d61c2cd057fcd1ea1e913f1f94a57b2c2f5dbd99a0ba89a7c674d643f54","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260989977235456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6070,"pr_state":"open","pr_sha":"34de8d334e30b0d09e281f7a85ee75d2e4eb369f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:41:05Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6fde5f0ca339e5b967e7885e2c18c0e92f8b280183fc5380c700c3781a5d51b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260989977235456","occurred_at":"2026-08-25T14:41:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"427763772c2cd234e542a70b64cad19c14d6c447","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:38:17Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"52d8d8451bcf45a5713700e04608b04d55fb1019c513ea44a60c0c9a6e9eb10a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"5495c7e40b7d6f88679370899209247dd34edae0e049192d2e9ffdc18b85813d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","failed_at":"e2e","signature_id":"40844011a6d47c6fdea97725589c77bfe98a0d94d59281c86c002f1a0792c273","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-sr4hjt444ntc/cluster-candidate-4-20-dvjvdg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018fc660\u003e: \n failed to create HCP cluster cluster-candidate-4-20-dvjvdg: failed waiting for cluster=\"cluster-candidate-4-20-dvjvdg\" in resourcegroup=\"rg-candidate-4-20-sr4hjt444ntc\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17e06728-cb97-46af-9b26-ca0125fb2f5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17e06728-cb97-46af-9b26-ca0125fb2f5d\",\n \"name\": \"17e06728-cb97-46af-9b26-ca0125fb2f5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:40:51.829519403Z\",\n \"endTime\": \"2026-08-25T15:59:58.300021131Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-sr4hjt444ntc/cluster-candidate-4-20-dvjvdg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018fc660\u003e: \n failed to create HCP cluster cluster-candidate-4-20-dvjvdg: failed waiting for cluster=\"cluster-candidate-4-20-dvjvdg\" in resourcegroup=\"rg-candidate-4-20-sr4hjt444ntc\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17e06728-cb97-46af-9b26-ca0125fb2f5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17e06728-cb97-46af-9b26-ca0125fb2f5d\",\n \"name\": \"17e06728-cb97-46af-9b26-ca0125fb2f5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:40:51.829519403Z\",\n \"endTime\": \"2026-08-25T15:59:58.300021131Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5e6d376a57b1d9779f666d7d22f96fc55793e95663c464a10be642fa1f0a2992","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","failed_at":"e2e","signature_id":"cf8d75a064f159907166271d9eeaa685334da9829b43b0bb83f85eba1df8e3e4","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.22.77:37397-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.22.77:37397-\u003e172.30.0.10:53: i/o timeout","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06","failure_pattern":"failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"\u003curl\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4."}},{"row_id":"a027cbe41391a7478b760e3e1422d02aeb93997906bca1eb0d337f56e8e131fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","failed_at":"e2e","signature_id":"2b07939ee9bb49277e2d643f0c99927b263772f00f1ecc4f749207d5a2800b2e","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8340\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-qhbwgqnksltm\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"name\": \"549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:41:53.637919079Z\",\n \"endTime\": \"2026-08-25T16:00:58.269163772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8340\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-qhbwgqnksltm\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"name\": \"549edf52-d404-4f5d-b33e-34f0219e1814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:41:53.637919079Z\",\n \"endTime\": \"2026-08-25T16:00:58.269163772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d188724e2ad8b2ff0009817ae8838a4d0cce4a292b1f33e9f843e12ff24e190c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f9ba61be5a69af64c20cd1c444d193114e6ad32f1ea119e727de88506923af3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","failed_at":"alert","signature_id":"ba0d4cca986340026ad28b58a3d7254c20c5a9e8dbe174366e6c0feb810b455d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (1 unknown, 19 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:55:35Z\n Ended: 2026-08-25T16:14:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9625216-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdmj1cccodu4opnletdsdsq44jkjroi-a5s1d1v8k8o6t3d\", pod=\"router-57477b9b98-d4bgr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdmj1cccodu4opnletdsdsq44jkjroi-a5s1d1v8k8o6t3d/router-57477b9b98-d4bgr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d188724e2ad8b2ff0009817ae8838a4d0cce4a292b1f33e9f843e12ff24e190c","db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260182896676864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6661,"pr_state":"open","pr_sha":"c54c32859a070847c08be4f9253dfcaacc383189","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:37:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"94a3b1f7645ab1c5c33535bd5368810b8a6da8ad55a9bbac868938f516aff3b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260182896676864","occurred_at":"2026-08-25T14:37:54Z","failed_at":"provision","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-25T15:21:22.056Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T15:21:22.495Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-25T15:21:22.967Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6676864-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac\ntime=2026-08-25T15:21:23.080Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6676864-mgmt-1/providers/Microsoft.Resources/deployments/5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac/operationStatuses/08584139356027184199\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"ca40fcb4ad5f5755d88716d21344ddcd3b294d47","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:29:37Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"7d9d83b783f161ab4e1a711c0f07d39d46a71c6ed7dc8c2e3fcb1e135351cd4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","occurred_at":"2026-08-25T14:29:37Z","failed_at":"alert","signature_id":"c9fd5e19ecb8a19404c13852abed9dafbdc292ec71487776299a01961be38d2a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:54:41Z\n Ended: 2026-08-25T16:00:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0144640-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdmbg6k1nbhvearl4c6d0hnook59uvs-v20260901\", pod=\"openshift-apiserver-79fd97c87b-6mplf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdmbg6k1nbhvearl4c6d0hnook59uvs-v20260901/openshift-apiserver-79fd97c87b-6mplf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ec3f3a3ec8998965f8401b71c293b0244dd523ec503d652103c09d189bb44184","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","occurred_at":"2026-08-25T14:29:37Z","failed_at":"alert","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:28:49Z\n Ended: 2026-08-25T15:35:49Z\n Severity: Sev3\n Labels: alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0144640-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257519584940032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:27:19Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"83e8dc4948bab7236cee65c5702a3fa61ec563c5742ba4478dfdca098886093e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257519584940032","occurred_at":"2026-08-25T14:27:19Z","failed_at":"e2e","signature_id":"1420b0f2188f90f92326018bb88dfc5b387aae62ec7dcbd9fa65850c57c8d47e","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cce280\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rg4h7xfrjp8d\" to finish creating: GET https://rp.j4940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"name\": \"9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:24:41.72360651Z\",\n \"endTime\": \"2026-08-25T15:43:49.738296088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, dns-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cce280\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rg4h7xfrjp8d\" to finish creating: GET https://rp.j4940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"name\": \"9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:24:41.72360651Z\",\n \"endTime\": \"2026-08-25T15:43:49.738296088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, dns-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257335450800128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6669,"pr_state":"open","pr_sha":"5c0ce63e7ccb9720b19764274e1ec873a56d9880","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:26:35Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b0292b01f1c7e9bc9fde3d1b98e99177a417a7b1862f07b5de9bff99e3634115","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257335450800128","occurred_at":"2026-08-25T14:26:35Z","failed_at":"e2e","signature_id":"f0c15c8af4ed67bd702e14b46d290c6f294cf901457e9886c4b4f48671c2cecc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vq9mz9ctfdtr/cluster-candidate-4-22-lq4rs9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00183a120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-lq4rs9: failed waiting for cluster=\"cluster-candidate-4-22-lq4rs9\" in resourcegroup=\"rg-candidate-4-22-vq9mz9ctfdtr\" to finish creating: GET https://rp.j0800128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476cd9df-c870-4d67-9b4d-2d1a52e94c91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476cd9df-c870-4d67-9b4d-2d1a52e94c91\",\n \"name\": \"476cd9df-c870-4d67-9b4d-2d1a52e94c91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:35.691904584Z\",\n \"endTime\": \"2026-08-25T15:45:42.445862232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vq9mz9ctfdtr/cluster-candidate-4-22-lq4rs9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00183a120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-lq4rs9: failed waiting for cluster=\"cluster-candidate-4-22-lq4rs9\" in resourcegroup=\"rg-candidate-4-22-vq9mz9ctfdtr\" to finish creating: GET https://rp.j0800128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476cd9df-c870-4d67-9b4d-2d1a52e94c91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476cd9df-c870-4d67-9b4d-2d1a52e94c91\",\n \"name\": \"476cd9df-c870-4d67-9b4d-2d1a52e94c91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:26:35.691904584Z\",\n \"endTime\": \"2026-08-25T15:45:42.445862232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092254294437793792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6661,"pr_state":"open","pr_sha":"75a66f437d6db5991dcd96efea40631bd9e314e8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:14:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"20709c03268775db100a1e60ab42941fab59251600ee59cc7ae6d9892996f98a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092254294437793792","occurred_at":"2026-08-25T14:14:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092254086249320448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6638,"pr_state":"open","pr_sha":"c403afe8d857fa8c76e2da31fc1c9d3952ca11b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:13:40Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9cadfbcf344a24b39f1614f4982fd26bcb4282a81caf8054b8a4e9d8b3b5ca60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092254086249320448","occurred_at":"2026-08-25T14:13:40Z","failed_at":"alert","signature_id":"6fc405c92e08fa1576ff900b57b0c002219d66eb9e455d1fd3d73886dc969d02","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (1 unknown, 19 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:12:14Z\n Ended: 2026-08-25T15:18:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9320448-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlunpahc0163917kblhfvqqfmg5rbf-u9s4n8y1l2f2f0w\", pod=\"cluster-network-operator-64b9458898-mnlcb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlunpahc0163917kblhfvqqfmg5rbf-u9s4n8y1l2f2f0w/cluster-network-operator-64b9458898-mnlcb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6587,"pr_state":"open","pr_sha":"f0ff1a7ad41be75beee4672fe337632d6d4a23ff","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:11:22Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6c35ec356ef27586a67be5b9ed32781bf6babb0e0f9335a000e32f2948f637ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","occurred_at":"2026-08-25T14:11:22Z","failed_at":"alert","signature_id":"2cc3c6061c407d3c74ad5325b6c2ccb6ae669b0fee5d48f4299ed1a3a6b6f5c4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 21 time(s) (1 unknown, 20 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:29:40Z\n Ended: 2026-08-25T15:35:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9425920-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdm06nri3tscpmbts3s7sj19ggcvld4-ea-listlkvss6\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdm06nri3tscpmbts3s7sj19ggcvld4-ea-listlkvss6/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ebcfda02c459613720a7348666b5b43b4cc05fca0ae21443fddeb5cc28cfe785","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","occurred_at":"2026-08-25T14:11:22Z","failed_at":"e2e","signature_id":"4d865de5fe9dfc533ed0216b192b218fb0191025036e4ded553b013e74cb84a3","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de6180\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-w7q75thqw7pj\" to finish creating: GET https://rp.j9425920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b0c7885-5533-4919-9374-a469f0b9ceb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b0c7885-5533-4919-9374-a469f0b9ceb9\",\n \"name\": \"9b0c7885-5533-4919-9374-a469f0b9ceb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:00:00.132611502Z\",\n \"endTime\": \"2026-08-25T15:19:06.963030567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de6180\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-w7q75thqw7pj\" to finish creating: GET https://rp.j9425920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b0c7885-5533-4919-9374-a469f0b9ceb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b0c7885-5533-4919-9374-a469f0b9ceb9\",\n \"name\": \"9b0c7885-5533-4919-9374-a469f0b9ceb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T15:00:00.132611502Z\",\n \"endTime\": \"2026-08-25T15:19:06.963030567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"7eff23c12e4a99825ffd97ce671ba00e24716cc4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:10:01Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0e44fd772c552b1ae3622a52af178ae19bdcba27c57b6e0285a13ac8d30263dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"0238da5b4ee608676a62f88216ac5a7d91c16da91f1ec76842258c4d72bebf4e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-7bbblv\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f0940\u003e: \n failed to create HCP cluster cluster-zstream-4-20-7bbblv: failed waiting for cluster=\"cluster-zstream-4-20-7bbblv\" in resourcegroup=\"rg-zstream-upgrade-4-20-t8z5vnqflfbk\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6f56725-5efb-4b8e-89e7-68c89ae9fd73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6f56725-5efb-4b8e-89e7-68c89ae9fd73\",\n \"name\": \"f6f56725-5efb-4b8e-89e7-68c89ae9fd73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:45.994759307Z\",\n \"endTime\": \"2026-08-25T15:12:49.360735646Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-7bbblv\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f0940\u003e: \n failed to create HCP cluster cluster-zstream-4-20-7bbblv: failed waiting for cluster=\"cluster-zstream-4-20-7bbblv\" in resourcegroup=\"rg-zstream-upgrade-4-20-t8z5vnqflfbk\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6f56725-5efb-4b8e-89e7-68c89ae9fd73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6f56725-5efb-4b8e-89e7-68c89ae9fd73\",\n \"name\": \"f6f56725-5efb-4b8e-89e7-68c89ae9fd73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:45.994759307Z\",\n \"endTime\": \"2026-08-25T15:12:49.360735646Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"20973af0bee2523436537429f242d5506bc94d1277235d3051c36306f1b18289","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"b2eebc2a537ba9b4668bce19ae91591fac3e22a5f8939d98e9175fbce444f838","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f101e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ghf78tzvlbsq\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72f94cb8-df3d-4852-ac78-087d67b17c38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72f94cb8-df3d-4852-ac78-087d67b17c38\",\n \"name\": \"72f94cb8-df3d-4852-ac78-087d67b17c38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:56.398337094Z\",\n \"endTime\": \"2026-08-25T15:12:59.401611971Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f101e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ghf78tzvlbsq\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72f94cb8-df3d-4852-ac78-087d67b17c38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72f94cb8-df3d-4852-ac78-087d67b17c38\",\n \"name\": \"72f94cb8-df3d-4852-ac78-087d67b17c38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:56.398337094Z\",\n \"endTime\": \"2026-08-25T15:12:59.401611971Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60a5c1b36e80a8820e97b2f7e6c05d837c88838be1a9815dca066010f93d0aa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"a62be0324e4af4aa551666eb60b0b9159ceb611d57868bbf7b768ffe258fbfbc","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d6380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-xwfdq94j4nwz\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c786f507-3037-418d-9e46-eab4b7d0f760\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c786f507-3037-418d-9e46-eab4b7d0f760\",\n \"name\": \"c786f507-3037-418d-9e46-eab4b7d0f760\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:56.198919711Z\",\n \"endTime\": \"2026-08-25T15:12:59.311902322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d6380\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-xwfdq94j4nwz\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c786f507-3037-418d-9e46-eab4b7d0f760\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c786f507-3037-418d-9e46-eab4b7d0f760\",\n \"name\": \"c786f507-3037-418d-9e46-eab4b7d0f760\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:56.198919711Z\",\n \"endTime\": \"2026-08-25T15:12:59.311902322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a124e5d7ae4134f7dc5b93ab6c21db39110d7960b1674daf0d9cdc468efef0d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"9f0220de1cfd60808a1dbdb143dba6ec11917303d1f81188a2549135945ac704","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b67620\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-wgbj289pcnlt\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\",\n \"name\": \"37f858c7-a07c-43e4-a020-589372bc5756\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:06.405705309Z\",\n \"endTime\": \"2026-08-25T15:13:09.352514019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b67620\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-wgbj289pcnlt\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\",\n \"name\": \"37f858c7-a07c-43e4-a020-589372bc5756\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:06.405705309Z\",\n \"endTime\": \"2026-08-25T15:13:09.352514019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a26f6e1e34171f63d2d6f8d0cb731378b1975f85cd85a730c34febee6a564eaf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"672eae11be59af76cbd1f197619054a7417629804d462d7487cf8b975b64fdd2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pk8mtfhvgljp/cluster-candidate-4-20-bxb275 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dc8c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-bxb275: failed waiting for cluster=\"cluster-candidate-4-20-bxb275\" in resourcegroup=\"rg-candidate-4-20-pk8mtfhvgljp\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7178f3ab-0cd5-4d88-98c6-7dbda7360f2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7178f3ab-0cd5-4d88-98c6-7dbda7360f2e\",\n \"name\": \"7178f3ab-0cd5-4d88-98c6-7dbda7360f2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:55.591878738Z\",\n \"endTime\": \"2026-08-25T15:12:59.329954081Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pk8mtfhvgljp/cluster-candidate-4-20-bxb275 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dc8c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-bxb275: failed waiting for cluster=\"cluster-candidate-4-20-bxb275\" in resourcegroup=\"rg-candidate-4-20-pk8mtfhvgljp\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7178f3ab-0cd5-4d88-98c6-7dbda7360f2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7178f3ab-0cd5-4d88-98c6-7dbda7360f2e\",\n \"name\": \"7178f3ab-0cd5-4d88-98c6-7dbda7360f2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:55.591878738Z\",\n \"endTime\": \"2026-08-25T15:12:59.329954081Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a71ad32fd05b21f5d0c968b5ca2afc63f2059d9d8468a00d4ebb1726ab5167a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"5aee83e4a720dd3ebb6047456af4c05c8c39902dcd0a2d8df1e2007ebdf2f0c2","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-ms5drw at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001050240\u003e: \n failed to create HCP cluster np-skip-minor-ms5drw: failed waiting for cluster=\"np-skip-minor-ms5drw\" in resourcegroup=\"rg-np-skip-minor-ms5drw-v9s5v98ch8vm\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07347ef0-e748-4346-97d6-c0ec7315b19c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07347ef0-e748-4346-97d6-c0ec7315b19c\",\n \"name\": \"07347ef0-e748-4346-97d6-c0ec7315b19c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:58.520345438Z\",\n \"endTime\": \"2026-08-25T15:12:59.410612687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-ms5drw at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001050240\u003e: \n failed to create HCP cluster np-skip-minor-ms5drw: failed waiting for cluster=\"np-skip-minor-ms5drw\" in resourcegroup=\"rg-np-skip-minor-ms5drw-v9s5v98ch8vm\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07347ef0-e748-4346-97d6-c0ec7315b19c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07347ef0-e748-4346-97d6-c0ec7315b19c\",\n \"name\": \"07347ef0-e748-4346-97d6-c0ec7315b19c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:58.520345438Z\",\n \"endTime\": \"2026-08-25T15:12:59.410612687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c41f2b66ae31310c169f2c9284e27d88fc4066511ad02674d0560d2a56fe3e69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"020633b5339d75f1df2cb64c5d4e04f6ce5d65297c8f1dc716fac948850492b0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8zv2whbhlk5z/cluster-candidate-5-0-lxbbwb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114a5c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-lxbbwb: failed waiting for cluster=\"cluster-candidate-5-0-lxbbwb\" in resourcegroup=\"rg-candidate-5-0-8zv2whbhlk5z\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca4022d-9521-44e9-8378-8f7a085e07b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca4022d-9521-44e9-8378-8f7a085e07b0\",\n \"name\": \"5ca4022d-9521-44e9-8378-8f7a085e07b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:56.011376402Z\",\n \"endTime\": \"2026-08-25T15:12:59.566489384Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8zv2whbhlk5z/cluster-candidate-5-0-lxbbwb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114a5c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-lxbbwb: failed waiting for cluster=\"cluster-candidate-5-0-lxbbwb\" in resourcegroup=\"rg-candidate-5-0-8zv2whbhlk5z\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca4022d-9521-44e9-8378-8f7a085e07b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca4022d-9521-44e9-8378-8f7a085e07b0\",\n \"name\": \"5ca4022d-9521-44e9-8378-8f7a085e07b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:53:56.011376402Z\",\n \"endTime\": \"2026-08-25T15:12:59.566489384Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c7aa6cbd27dd4e760ad97a69c79b7b8b760503e0ff2d7a2105d46cc6cb291a50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","failed_at":"e2e","signature_id":"c3b3ef09c20cc547bca5325c7f41e4dd0c4a174e6693ff2418f6a450ed448c6b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-9v2fm7\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc760\u003e: \n failed to create HCP cluster cluster-zstream-4-22-9v2fm7: failed waiting for cluster=\"cluster-zstream-4-22-9v2fm7\" in resourcegroup=\"rg-zstream-upgrade-4-22-qzxxgjsj26qb\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"name\": \"02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:16.06655817Z\",\n \"endTime\": \"2026-08-25T15:13:19.307227089Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-9v2fm7\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc760\u003e: \n failed to create HCP cluster cluster-zstream-4-22-9v2fm7: failed waiting for cluster=\"cluster-zstream-4-22-9v2fm7\" in resourcegroup=\"rg-zstream-upgrade-4-22-qzxxgjsj26qb\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"name\": \"02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:54:16.06655817Z\",\n \"endTime\": \"2026-08-25T15:13:19.307227089Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253020946436096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:09:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"dc305d2aab2cc8cc5877ce0b11e897083c6b7c94f79340214c05a7e8155bf8da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253020946436096","occurred_at":"2026-08-25T14:09:13Z","failed_at":"provision","signature_id":"3b416c8626d510d6d228f2b4d85a59b118ef1a435941087523bd321e4dac7444","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-25T14:41:26.287Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T14:41:26.667Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2\ntime=2026-08-25T14:41:27.045Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 deployment=3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6436096-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b\ntime=2026-08-25T14:41:27.116Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6436096-mgmt-2/providers/Microsoft.Resources/deployments/3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b/operationStatuses/08584139379985620464\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6630,"pr_state":"open","pr_sha":"53966db5bb00eb465ea4270e8fed068375360c6c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T14:01:22Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"04038603adbe905fee1521ab3411467fe5b6c7a55f2853823c8d123f69b55479","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"e2e","signature_id":"279d843775eb67f9cc4f7628611a62e9cdb3df14488565ed034ba6ce68dad623","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee3660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pfd8fc762xff\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"name\": \"c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:47:46.321950941Z\",\n \"endTime\": \"2026-08-25T15:06:53.075223567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee3660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pfd8fc762xff\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"name\": \"c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:47:46.321950941Z\",\n \"endTime\": \"2026-08-25T15:06:53.075223567Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"09f986eeaef9ef03e89acc8ef57bf3aa4f7820de8da84049b2bc414381ff7c7f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"08376e0d318c8d9f0177d77d214d72917cc2f71b4ec167c6844718dc62ce5959","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"e2e","signature_id":"0dd4ca7c298b5c4afeaab0da7cacaeb92e7380f6b13a23bb92ff6fc133819fd1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-gnzmjzqw2kss/cluster-candidate-5-0-bbkwfd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e640\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bbkwfd: failed waiting for cluster=\"cluster-candidate-5-0-bbkwfd\" in resourcegroup=\"rg-candidate-5-0-gnzmjzqw2kss\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d991771-9053-4970-8d4f-39b2d38bc505\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d991771-9053-4970-8d4f-39b2d38bc505\",\n \"name\": \"4d991771-9053-4970-8d4f-39b2d38bc505\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:46:45.526857015Z\",\n \"endTime\": \"2026-08-25T15:05:53.073478915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-gnzmjzqw2kss/cluster-candidate-5-0-bbkwfd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e640\u003e: \n failed to create HCP cluster cluster-candidate-5-0-bbkwfd: failed waiting for cluster=\"cluster-candidate-5-0-bbkwfd\" in resourcegroup=\"rg-candidate-5-0-gnzmjzqw2kss\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d991771-9053-4970-8d4f-39b2d38bc505\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d991771-9053-4970-8d4f-39b2d38bc505\",\n \"name\": \"4d991771-9053-4970-8d4f-39b2d38bc505\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:46:45.526857015Z\",\n \"endTime\": \"2026-08-25T15:05:53.073478915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1747e2c4b7ae7e9565b2c73cea7efdad74f491f263c93ba2bd60f29c1d1db3f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"e2e","signature_id":"d5aeb36659e2fe6d42bc3c374436c19a2f23ca86b7d193e2d4e494e04f3f5eda","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109e740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-225vnhm84lkc\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7eb9a5b-8f23-46ff-8dff-b70b00b25966\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7eb9a5b-8f23-46ff-8dff-b70b00b25966\",\n \"name\": \"c7eb9a5b-8f23-46ff-8dff-b70b00b25966\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:48:06.4361502Z\",\n \"endTime\": \"2026-08-25T15:07:13.076236238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109e740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-225vnhm84lkc\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7eb9a5b-8f23-46ff-8dff-b70b00b25966\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7eb9a5b-8f23-46ff-8dff-b70b00b25966\",\n \"name\": \"c7eb9a5b-8f23-46ff-8dff-b70b00b25966\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:48:06.4361502Z\",\n \"endTime\": \"2026-08-25T15:07:13.076236238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2316922f4cfb3c7378ca5bb1604b7ea0983787e665b7485101b86dd999e2deb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"3dd1f4b54cbed815ed27b3ab048e2cd17dcdd5df732a90f013c6ea10a6e06040","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"alert","signature_id":"0a3222b13ca8e18b09f294cc8bf6c58f2b714b4f68671591fc81d113df214527","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s) (1 unknown, 23 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T15:00:54Z\n Ended: 2026-08-25T15:20:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9836800-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlpln0oubfmm7l5rkf2udfse2o25ud-o6g6g1g5g3v3m3u\", pod=\"router-7c5b776b9c-4765d\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlpln0oubfmm7l5rkf2udfse2o25ud-o6g6g1g5g3v3m3u/router-7c5b776b9c-4765d has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5d0a9f56c5f9d158a2b33a0d0e30729d1f6fc299e5d4e1e8f48030fadd66692a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"e2e","signature_id":"25985bba716906e38dc1ab2e46edefe47ee71312f5d30ecca45c85fd1a612dae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-dlh9fkj7trzd/cluster-candidate-4-21-59tf2h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cba160\u003e: \n failed to create HCP cluster cluster-candidate-4-21-59tf2h: failed waiting for cluster=\"cluster-candidate-4-21-59tf2h\" in resourcegroup=\"rg-candidate-4-21-dlh9fkj7trzd\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26c41da-0377-42e1-a30e-6696aceb0e40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26c41da-0377-42e1-a30e-6696aceb0e40\",\n \"name\": \"e26c41da-0377-42e1-a30e-6696aceb0e40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:46:46.759513984Z\",\n \"endTime\": \"2026-08-25T15:05:52.992371742Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-dlh9fkj7trzd/cluster-candidate-4-21-59tf2h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cba160\u003e: \n failed to create HCP cluster cluster-candidate-4-21-59tf2h: failed waiting for cluster=\"cluster-candidate-4-21-59tf2h\" in resourcegroup=\"rg-candidate-4-21-dlh9fkj7trzd\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26c41da-0377-42e1-a30e-6696aceb0e40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26c41da-0377-42e1-a30e-6696aceb0e40\",\n \"name\": \"e26c41da-0377-42e1-a30e-6696aceb0e40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:46:46.759513984Z\",\n \"endTime\": \"2026-08-25T15:05:52.992371742Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7c954db3e73d2eba946a680a3587cec9717e49659bffee3d1e6a5553102ceb70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","failed_at":"e2e","signature_id":"19f2511c94e5b6ca21cc71bd63aabc9053c7c653b42b617ec879ca395e30580c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-ldcljf2ctfmx/cluster-candidate-4-22-bzcn4d should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec0120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bzcn4d: failed waiting for cluster=\"cluster-candidate-4-22-bzcn4d\" in resourcegroup=\"rg-candidate-4-22-ldcljf2ctfmx\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26994133-5a3a-4dc6-8825-a517ba8b6492\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26994133-5a3a-4dc6-8825-a517ba8b6492\",\n \"name\": \"26994133-5a3a-4dc6-8825-a517ba8b6492\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:46:46.263967961Z\",\n \"endTime\": \"2026-08-25T15:05:53.038982489Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-ldcljf2ctfmx/cluster-candidate-4-22-bzcn4d should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec0120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bzcn4d: failed waiting for cluster=\"cluster-candidate-4-22-bzcn4d\" in resourcegroup=\"rg-candidate-4-22-ldcljf2ctfmx\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26994133-5a3a-4dc6-8825-a517ba8b6492\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26994133-5a3a-4dc6-8825-a517ba8b6492\",\n \"name\": \"26994133-5a3a-4dc6-8825-a517ba8b6492\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:46:46.263967961Z\",\n \"endTime\": \"2026-08-25T15:05:53.038982489Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["09f986eeaef9ef03e89acc8ef57bf3aa4f7820de8da84049b2bc414381ff7c7f","0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:55:13Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"114be452c12feed5c726f33387080787a24bd8b233434938162ef9e7a573305c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","failed_at":"e2e","signature_id":"c1c5c1541665e5683a94093cd7f6588bce5f0027f1e226c12d34e5a06ef243a7","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6d3c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-5rgd5mxghdpb\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf3f32cf-b93f-4884-bc0a-e3893898bf0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf3f32cf-b93f-4884-bc0a-e3893898bf0c\",\n \"name\": \"cf3f32cf-b93f-4884-bc0a-e3893898bf0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:21.418735516Z\",\n \"endTime\": \"2026-08-25T14:59:24.842684505Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6d3c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-5rgd5mxghdpb\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf3f32cf-b93f-4884-bc0a-e3893898bf0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf3f32cf-b93f-4884-bc0a-e3893898bf0c\",\n \"name\": \"cf3f32cf-b93f-4884-bc0a-e3893898bf0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:21.418735516Z\",\n \"endTime\": \"2026-08-25T14:59:24.842684505Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"14322491a18c180572ed8432f200600a5d85617b80fc06b3421509636e1e469c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","failed_at":"e2e","signature_id":"02ce83b411d6d3eadebb040a79dc7e0fc19f5dc8037f2c40f8fbbf51c121a5e0","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee4d80\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h2mbhr4mc5lp\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c31876ee-87ff-43fd-a5d5-bd1cf0cac5a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c31876ee-87ff-43fd-a5d5-bd1cf0cac5a9\",\n \"name\": \"c31876ee-87ff-43fd-a5d5-bd1cf0cac5a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:53.432001681Z\",\n \"endTime\": \"2026-08-25T14:59:54.838826092Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee4d80\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h2mbhr4mc5lp\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c31876ee-87ff-43fd-a5d5-bd1cf0cac5a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c31876ee-87ff-43fd-a5d5-bd1cf0cac5a9\",\n \"name\": \"c31876ee-87ff-43fd-a5d5-bd1cf0cac5a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:53.432001681Z\",\n \"endTime\": \"2026-08-25T14:59:54.838826092Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"43458d927d135e4cfc3f00eed86a7b42dff9fa9180f958557484332bf3b70396","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","failed_at":"alert","signature_id":"7d256492ee5379d7f038bf539080c373cfbf081cb07ec0c35738f57cdb906cc2","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 15 time(s) (4 unknown, 11 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T14:52:03Z\n Ended: 2026-08-25T15:12:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3839232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmo7baqls1gop29vgsct7kabvoth8-g0w4c9d3q6r2r1m\", pod=\"router-974f5ddb-2dgs2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmo7baqls1gop29vgsct7kabvoth8-g0w4c9d3q6r2r1m/router-974f5ddb-2dgs2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T14:52:03Z\n Ended: 2026-08-25T15:13:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3839232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmtn9ovk9tpsm9j2ft6snivi04oqe-a2p1p1g5j3i3f3u\", pod=\"router-556f6fb4ff-nfxjj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmtn9ovk9tpsm9j2ft6snivi04oqe-a2p1p1g5j3i3f3u/router-556f6fb4ff-nfxjj has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T14:53:02Z\n Ended: 2026-08-25T15:12:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3839232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmlash2aq785fmi7mq3rc7dfln3rc-k6a8t8m0v1f0j5x\", pod=\"router-6bd4d54bc8-vsx2s\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmlash2aq785fmi7mq3rc7dfln3rc-k6a8t8m0v1f0j5x/router-6bd4d54bc8-vsx2s has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T14:55:02Z\n Ended: 2026-08-25T15:12:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3839232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmo1em3u37eqkj6qat0evo6kvf8qt-v20260901\", pod=\"router-695b86fdc7-zr4fd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmo1em3u37eqkj6qat0evo6kvf8qt-v20260901/router-695b86fdc7-zr4fd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"91b4422e71a1ee1cc63eb66934b86c4926895e0141de4aa27bca95808a13a596","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","failed_at":"e2e","signature_id":"59eb8e67410e16bbb2ec63be568c198772ad94ee9f1c1cbda4a99f3c7987422a","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00094e440\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cjmmvcxplkm2\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/729aa40b-70d3-4080-bbfb-0d103a608dbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/729aa40b-70d3-4080-bbfb-0d103a608dbd\",\n \"name\": \"729aa40b-70d3-4080-bbfb-0d103a608dbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:31.102654223Z\",\n \"endTime\": \"2026-08-25T14:59:34.736723772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00094e440\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cjmmvcxplkm2\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/729aa40b-70d3-4080-bbfb-0d103a608dbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/729aa40b-70d3-4080-bbfb-0d103a608dbd\",\n \"name\": \"729aa40b-70d3-4080-bbfb-0d103a608dbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:31.102654223Z\",\n \"endTime\": \"2026-08-25T14:59:34.736723772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c3fc2d205e7020c8cbaeb00bf493d4a6b8d3cc4a84f7ea7eb06859c769f56485","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","failed_at":"e2e","signature_id":"7bd5b0bf2a08febff61bb7112c5fa60448aafb6f03c45e0ecefda98be25fcc90","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dnw5jvjfdwmc/cluster-candidate-5-0-272q64 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f74280\u003e: \n failed to create HCP cluster cluster-candidate-5-0-272q64: failed waiting for cluster=\"cluster-candidate-5-0-272q64\" in resourcegroup=\"rg-candidate-5-0-dnw5jvjfdwmc\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e65c60e0-1cb1-4ac7-85e2-b7ecad902d21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e65c60e0-1cb1-4ac7-85e2-b7ecad902d21\",\n \"name\": \"e65c60e0-1cb1-4ac7-85e2-b7ecad902d21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:31.01500763Z\",\n \"endTime\": \"2026-08-25T14:59:34.82035333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dnw5jvjfdwmc/cluster-candidate-5-0-272q64 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f74280\u003e: \n failed to create HCP cluster cluster-candidate-5-0-272q64: failed waiting for cluster=\"cluster-candidate-5-0-272q64\" in resourcegroup=\"rg-candidate-5-0-dnw5jvjfdwmc\" to finish creating: GET https://rp.j3839232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e65c60e0-1cb1-4ac7-85e2-b7ecad902d21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e65c60e0-1cb1-4ac7-85e2-b7ecad902d21\",\n \"name\": \"e65c60e0-1cb1-4ac7-85e2-b7ecad902d21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:31.01500763Z\",\n \"endTime\": \"2026-08-25T14:59:34.82035333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6677,"pr_state":"open","pr_sha":"0d881db792c6eb930fa946fe3df7da0fca683688","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:52:37Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"64d725ce71ec66cf37121083aee97ccdcaa56613cd287955e8f6547c9ebf253a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","failed_at":"e2e","signature_id":"ebdc0b91a773f55107daffc829136d740ef7567c97d135876b3ab2f729e95efe","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000beb2a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sn5zc7drh7f4\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34aa8a23-7309-49a6-83bf-51c69ef9dc4e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34aa8a23-7309-49a6-83bf-51c69ef9dc4e\",\n \"name\": \"34aa8a23-7309-49a6-83bf-51c69ef9dc4e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:37.489505349Z\",\n \"endTime\": \"2026-08-25T14:59:39.545936275Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000beb2a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sn5zc7drh7f4\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34aa8a23-7309-49a6-83bf-51c69ef9dc4e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34aa8a23-7309-49a6-83bf-51c69ef9dc4e\",\n \"name\": \"34aa8a23-7309-49a6-83bf-51c69ef9dc4e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:37.489505349Z\",\n \"endTime\": \"2026-08-25T14:59:39.545936275Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8aacd3a9c2b7f845b08348216f64a3104f68589b7b2ad449fcc790e178109ec1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","failed_at":"alert","signature_id":"85d3fb950aa7a81bc280ad8111f550c48065298ea6e7ee23a30155e0451f6699","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s) (4 unknown, 8 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T14:51:31Z\n Ended: 2026-08-25T15:14:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8970496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmpgvl8u8krpi89rr9e9vj1a2ll6v-b2s3x1u9q6t8r9m\", pod=\"router-54677d69bb-mfrpm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmpgvl8u8krpi89rr9e9vj1a2ll6v-b2s3x1u9q6t8r9m/router-54677d69bb-mfrpm has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T14:51:31Z\n Ended: 2026-08-25T15:12:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8970496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmkk71i1fsmkh3032328vp5kjd3ik-a4u0o6g1z8e2w2h\", pod=\"router-6c6b797c88-snsbr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmkk71i1fsmkh3032328vp5kjd3ik-a4u0o6g1z8e2w2h/router-6c6b797c88-snsbr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T14:51:31Z\n Ended: 2026-08-25T15:12:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8970496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmkk71i1fsmkh3032328vp5kjd3ik-a4u0o6g1z8e2w2h\", pod=\"router-6c6b797c88-ssz4g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmkk71i1fsmkh3032328vp5kjd3ik-a4u0o6g1z8e2w2h/router-6c6b797c88-ssz4g has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T14:55:28Z\n Ended: 2026-08-25T15:28:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8970496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlmbrgto6n0h5o6073npkpf9ttvibr-l4a3s3s1m8o9h0a\", pod=\"router-85db99885d-5dtz8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlmbrgto6n0h5o6073npkpf9ttvibr-l4a3s3s1m8o9h0a/router-85db99885d-5dtz8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"9453d301876d93522355b9223f1da92d54c0ff49c75f1e87c41b2f55ef16acd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"e69f02b4e4e872e8a1170781da8bb82c418e85b8459f413b26ca68373eb63983","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","failed_at":"e2e","signature_id":"9de0b947dcfc7f945c30a376f6c6b3a6af42020e652c40b79da8b091bb68bf00","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-x6bg4m6clbjq/cluster-candidate-4-22-v8dh2g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001724ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-v8dh2g: failed waiting for cluster=\"cluster-candidate-4-22-v8dh2g\" in resourcegroup=\"rg-candidate-4-22-x6bg4m6clbjq\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\",\n \"name\": \"8b6be2ed-087d-4652-a133-bf375421a904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:17.286340435Z\",\n \"endTime\": \"2026-08-25T14:59:19.220463305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-x6bg4m6clbjq/cluster-candidate-4-22-v8dh2g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001724ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-v8dh2g: failed waiting for cluster=\"cluster-candidate-4-22-v8dh2g\" in resourcegroup=\"rg-candidate-4-22-x6bg4m6clbjq\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\",\n \"name\": \"8b6be2ed-087d-4652-a133-bf375421a904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:40:17.286340435Z\",\n \"endTime\": \"2026-08-25T14:59:19.220463305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec5696a5a0d53642ab9a97bd3cf9f05f9a1163bd79ec1248a958413fd7c32640","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","failed_at":"e2e","signature_id":"0fc6eaf247bde243614d205434c32f36a568acb9c7fa2017a78c81ce9292cbf1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017db260\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-b4vb4qmlbwh5\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dd6f132-b097-41ed-a77e-dcf30d4582e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dd6f132-b097-41ed-a77e-dcf30d4582e3\",\n \"name\": \"6dd6f132-b097-41ed-a77e-dcf30d4582e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:42:41.004926697Z\",\n \"endTime\": \"2026-08-25T15:01:49.208965203Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017db260\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-b4vb4qmlbwh5\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dd6f132-b097-41ed-a77e-dcf30d4582e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dd6f132-b097-41ed-a77e-dcf30d4582e3\",\n \"name\": \"6dd6f132-b097-41ed-a77e-dcf30d4582e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:42:41.004926697Z\",\n \"endTime\": \"2026-08-25T15:01:49.208965203Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5958,"pr_state":"open","pr_sha":"ea2dde0a91dc9ddee9fd9192846ece1b78645d33","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:34:16Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"479499dd658f153900291ad4bf9f3f2ecaee5baaa4ce372a744269df5432e3ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","failed_at":"e2e","signature_id":"52544ef6923df629cf38fa8f3d327d749d9b4fc26a68706642aec73ba7646ec7","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019edc0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-prhmmrf56mn9\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"name\": \"90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:42.014090967Z\",\n \"endTime\": \"2026-08-25T14:54:47.709528737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019edc0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-prhmmrf56mn9\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"name\": \"90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:42.014090967Z\",\n \"endTime\": \"2026-08-25T14:54:47.709528737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"537acba501385bda0b8a4cacce315187ed343a6bf4bb4e16c36e2104c13b92d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","failed_at":"alert","signature_id":"46fe202d87700d2141f9c24fa084e86f0be02779da003c0aadb556639c6b00c1","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 19 time(s) (3 unknown, 16 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T14:39:16Z\n Ended: 2026-08-25T14:45:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0715520-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlfhd8vrqvchs6lae3ufqe2gul010f-o9t7t3k4b1c5z3g\", pod=\"cluster-storage-operator-5bb478c9-c6qxc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlfhd8vrqvchs6lae3ufqe2gul010f-o9t7t3k4b1c5z3g/cluster-storage-operator-5bb478c9-c6qxc has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T14:39:16Z\n Ended: 2026-08-25T14:45:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0715520-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlfef9mpp5b2vqt1no14mtaeu69k6k-x0s6g2t5w9k7o6l\", pod=\"cluster-network-operator-9bbf74b9c-m29mf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlfef9mpp5b2vqt1no14mtaeu69k6k-x0s6g2t5w9k7o6l/cluster-network-operator-9bbf74b9c-m29mf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T14:41:16Z\n Ended: 2026-08-25T14:47:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0715520-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlfgs8aqldhp3lh5r1j2t1paf41f8t-y2s5d6o5w9j5f8c\", pod=\"cluster-network-operator-55cfccd85d-6v4zw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlfgs8aqldhp3lh5r1j2t1paf41f8t-y2s5d6o5w9j5f8c/cluster-network-operator-55cfccd85d-6v4zw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c663c056f35a16912bcc65c4f48ba1257b1c90bf661240924b52528c8d3adeec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","failed_at":"e2e","signature_id":"ba2c82cf6ae3a73e48ea720374e43d7b38d997b258802c349e2dac2f93c95ba4","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100f360\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-tjq5k8gxthnl\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/093b11cd-6fee-48cf-93d0-7cfd46366e65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/093b11cd-6fee-48cf-93d0-7cfd46366e65\",\n \"name\": \"093b11cd-6fee-48cf-93d0-7cfd46366e65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:06.613580324Z\",\n \"endTime\": \"2026-08-25T14:44:07.687483877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100f360\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-tjq5k8gxthnl\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/093b11cd-6fee-48cf-93d0-7cfd46366e65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/093b11cd-6fee-48cf-93d0-7cfd46366e65\",\n \"name\": \"093b11cd-6fee-48cf-93d0-7cfd46366e65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:06.613580324Z\",\n \"endTime\": \"2026-08-25T14:44:07.687483877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dde49bf5e2824abf4aa5b4eb0657bc573f2ca1954d6639202c042f77907de3ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","failed_at":"e2e","signature_id":"5310caabe4a3b378f21ce354c7c0c3aac76e1ac8d392c8d5eaecc0741c517da0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-6pvw75qcwr4c/cluster-candidate-4-22-42l2l9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c7c4c0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-42l2l9: failed waiting for cluster=\"cluster-candidate-4-22-42l2l9\" in resourcegroup=\"rg-candidate-4-22-6pvw75qcwr4c\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2fbc0167-dc96-49fd-9ffa-f3dfd96dd6ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2fbc0167-dc96-49fd-9ffa-f3dfd96dd6ca\",\n \"name\": \"2fbc0167-dc96-49fd-9ffa-f3dfd96dd6ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:16.547735183Z\",\n \"endTime\": \"2026-08-25T14:44:17.680374337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-6pvw75qcwr4c/cluster-candidate-4-22-42l2l9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c7c4c0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-42l2l9: failed waiting for cluster=\"cluster-candidate-4-22-42l2l9\" in resourcegroup=\"rg-candidate-4-22-6pvw75qcwr4c\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2fbc0167-dc96-49fd-9ffa-f3dfd96dd6ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2fbc0167-dc96-49fd-9ffa-f3dfd96dd6ca\",\n \"name\": \"2fbc0167-dc96-49fd-9ffa-f3dfd96dd6ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:16.547735183Z\",\n \"endTime\": \"2026-08-25T14:44:17.680374337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6660,"pr_state":"open","pr_sha":"a660b6e55fc0b5bbac8bc93588a1797519a9c76f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:26:03Z"},"failed_at":["alert","e2e"],"failed_test_count":15,"occurrences":[{"row_id":"15db537e4e747800cb0756a8e609c891491ddb2311f85ab8fc5e631d19cdcb30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"52e3a2d01cede408cbe7165d526befd88f9e9f4dbcdc8aaa12bb61402619c205","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006490a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2ppc5kwlqt8f\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba1dde59-7246-414a-86eb-207a9c26ebc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba1dde59-7246-414a-86eb-207a9c26ebc0\",\n \"name\": \"ba1dde59-7246-414a-86eb-207a9c26ebc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.940463847Z\",\n \"endTime\": \"2026-08-25T14:43:14.557069884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006490a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2ppc5kwlqt8f\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba1dde59-7246-414a-86eb-207a9c26ebc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba1dde59-7246-414a-86eb-207a9c26ebc0\",\n \"name\": \"ba1dde59-7246-414a-86eb-207a9c26ebc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.940463847Z\",\n \"endTime\": \"2026-08-25T14:43:14.557069884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"195b0c705ab08e3c3de0c170aae4de9fbfcd9e1ea6bea441d948f289898d294a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"abc8216563c683da1f6e16ae575f37dd3f305f261e135e096e5da20727d96771","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df4260\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-jtj7cdlldhgj\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc8b29bb-eb97-4d27-a878-9f36e4dc7eb1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc8b29bb-eb97-4d27-a878-9f36e4dc7eb1\",\n \"name\": \"dc8b29bb-eb97-4d27-a878-9f36e4dc7eb1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:41.424970771Z\",\n \"endTime\": \"2026-08-25T14:41:44.363811133Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df4260\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-jtj7cdlldhgj\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc8b29bb-eb97-4d27-a878-9f36e4dc7eb1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc8b29bb-eb97-4d27-a878-9f36e4dc7eb1\",\n \"name\": \"dc8b29bb-eb97-4d27-a878-9f36e4dc7eb1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:41.424970771Z\",\n \"endTime\": \"2026-08-25T14:41:44.363811133Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"32b016ae108875bfd21c739962799b705a30ce1f0dcae3cb0fb166fa932d5515","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"2f9c966df057075e45f28d9a86360c4da74e7efb2d31b2a981af2a97722900a4","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da43a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-52lwzkdvrvxq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"name\": \"1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.556002909Z\",\n \"endTime\": \"2026-08-25T14:42:14.793443561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da43a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-52lwzkdvrvxq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"name\": \"1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.556002909Z\",\n \"endTime\": \"2026-08-25T14:42:14.793443561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"53d2304323e4d00a752a65ff26b8ef1dfce65a855234c30b28775d75abe462f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"139f5384ed9c8675144c62f1bbdf582bf6ba3de5a1a2a7e69cafbaf9e1324971","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--vdwvqtzmlhhz/cluster-hs-pre-h4zwsj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efebc0\u003e: \n failed to create HCP cluster cluster-hs-pre-h4zwsj: failed waiting for cluster=\"cluster-hs-pre-h4zwsj\" in resourcegroup=\"rg-hs-pre--vdwvqtzmlhhz\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7be718c8-db1c-4834-a67f-c3902ee1577e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7be718c8-db1c-4834-a67f-c3902ee1577e\",\n \"name\": \"7be718c8-db1c-4834-a67f-c3902ee1577e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:26.49157749Z\",\n \"endTime\": \"2026-08-25T14:41:34.372350557Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--vdwvqtzmlhhz/cluster-hs-pre-h4zwsj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efebc0\u003e: \n failed to create HCP cluster cluster-hs-pre-h4zwsj: failed waiting for cluster=\"cluster-hs-pre-h4zwsj\" in resourcegroup=\"rg-hs-pre--vdwvqtzmlhhz\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7be718c8-db1c-4834-a67f-c3902ee1577e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7be718c8-db1c-4834-a67f-c3902ee1577e\",\n \"name\": \"7be718c8-db1c-4834-a67f-c3902ee1577e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:26.49157749Z\",\n \"endTime\": \"2026-08-25T14:41:34.372350557Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5e16d340520ee60336762b9ef1b2ad8e817f24101f273f6a9871bd87c5de2a1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"8bd3d86b83f9075e0c109be0d76439a6735162556e0e3886ba813816b230c868","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e160\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-xdpzglqjl7k5\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22657958-a1fd-4572-9f54-a3f4670af3f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22657958-a1fd-4572-9f54-a3f4670af3f8\",\n \"name\": \"22657958-a1fd-4572-9f54-a3f4670af3f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:10.9485512Z\",\n \"endTime\": \"2026-08-25T14:44:14.603331408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e160\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-xdpzglqjl7k5\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22657958-a1fd-4572-9f54-a3f4670af3f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22657958-a1fd-4572-9f54-a3f4670af3f8\",\n \"name\": \"22657958-a1fd-4572-9f54-a3f4670af3f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:10.9485512Z\",\n \"endTime\": \"2026-08-25T14:44:14.603331408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a907ea31544138b98034b32aa0a4510d82380d0cc2590ba915ba3d67c5d518a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"alert","signature_id":"142358f238029e754285919ac9ae3371cf3d7fcc3f75f1908d60371d3bae5134","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s) (9 unknown, 2 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T14:35:07Z\n Ended: 2026-08-25T14:59:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlejnfp7395pe7ttfrm2o3t3i5i0s1-private-kas\", pod=\"router-7666b5d5c4-zznx6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlejnfp7395pe7ttfrm2o3t3i5i0s1-private-kas/router-7666b5d5c4-zznx6 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T14:35:07Z\n Ended: 2026-08-25T14:52:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlejnfp7395pe7ttfrm2o3t3i5i0s1-private-kas\", pod=\"router-7666b5d5c4-kfsdq\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlejnfp7395pe7ttfrm2o3t3i5i0s1-private-kas/router-7666b5d5c4-kfsdq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T14:35:07Z\n Ended: 2026-08-25T14:53:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdle8r3r5ancl77mi9gtmfo4opktruc-s1p8l7t8j3h1w3u\", pod=\"router-d94cfcdbd-4nj4q\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdle8r3r5ancl77mi9gtmfo4opktruc-s1p8l7t8j3h1w3u/router-d94cfcdbd-4nj4q has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T14:36:07Z\n Ended: 2026-08-25T14:52:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdldvge2kpq5odr57p465kue54uh9fi-a9a5z8y1j0x9z6j\", pod=\"router-657fb7bdcf-tc592\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdldvge2kpq5odr57p465kue54uh9fi-a9a5z8y1j0x9z6j/router-657fb7bdcf-tc592 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T14:36:08Z\n Ended: 2026-08-25T14:52:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdle8svkin145smgvdv0pdgj1d7mcgh-p5b7o7v3h7a1b0f\", pod=\"router-5bc7d4c559-rpll4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdle8svkin145smgvdv0pdgj1d7mcgh-p5b7o7v3h7a1b0f/router-5bc7d4c559-rpll4 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T14:37:07Z\n Ended: 2026-08-25T14:43:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlf3jn7opj4balf1jd2v7mg84482e1-cilium-cluster\", pod=\"router-648dc7cb9b-8mv4d\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlf3jn7opj4balf1jd2v7mg84482e1-cilium-cluster/router-648dc7cb9b-8mv4d has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T14:37:07Z\n Ended: 2026-08-25T14:53:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlebjb4f830k7dqj7mi38s5u8unig7-l1h8m2e0e1k3j3a\", pod=\"router-756bd757b9-sxttr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlebjb4f830k7dqj7mi38s5u8unig7-l1h8m2e0e1k3j3a/router-756bd757b9-sxttr has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T14:37:07Z\n Ended: 2026-08-25T14:58:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdle7lje546dmg1m8eo3r2g0l88kh33-p4w5p2d5h6o0y9w\", pod=\"router-6df7459d5-h8fkt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdle7lje546dmg1m8eo3r2g0l88kh33-p4w5p2d5h6o0y9w/router-6df7459d5-h8fkt has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T14:37:08Z\n Ended: 2026-08-25T14:55:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7603200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlebjb4f830k7dqj7mi38s5u8unig7-l1h8m2e0e1k3j3a\", pod=\"router-756bd757b9-pmmhv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlebjb4f830k7dqj7mi38s5u8unig7-l1h8m2e0e1k3j3a/router-756bd757b9-pmmhv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"7200a625ac6f7bb05fc8b9ce7fc37d7ab9827d9f036b68a08bb5dafd801e7c2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"bccfe75fe11d0ea8f7728479404e3d27d3b6a70c50e8e5d0793d665e9d222fbf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-jprm6wpttrf7/cluster-candidate-5-0-n79n6k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5a120\u003e: \n failed to create HCP cluster cluster-candidate-5-0-n79n6k: failed waiting for cluster=\"cluster-candidate-5-0-n79n6k\" in resourcegroup=\"rg-candidate-5-0-jprm6wpttrf7\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5c105bf-67e2-4a17-aebe-38cc80b7408e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5c105bf-67e2-4a17-aebe-38cc80b7408e\",\n \"name\": \"b5c105bf-67e2-4a17-aebe-38cc80b7408e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:22.605993872Z\",\n \"endTime\": \"2026-08-25T14:41:24.396649573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-jprm6wpttrf7/cluster-candidate-5-0-n79n6k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5a120\u003e: \n failed to create HCP cluster cluster-candidate-5-0-n79n6k: failed waiting for cluster=\"cluster-candidate-5-0-n79n6k\" in resourcegroup=\"rg-candidate-5-0-jprm6wpttrf7\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5c105bf-67e2-4a17-aebe-38cc80b7408e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5c105bf-67e2-4a17-aebe-38cc80b7408e\",\n \"name\": \"b5c105bf-67e2-4a17-aebe-38cc80b7408e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:22.605993872Z\",\n \"endTime\": \"2026-08-25T14:41:24.396649573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8afa2fc8c721fcae97aa72234df16475cfb0fe69ad0c9ec4b86338ecefe3a149","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"3bedaad2d7a72d761e92c1a0c1322773388504505b8f7b830d9c172d097aab52","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-2n6qpzrjsvhw/cluster-candidate-4-21-zb4zw7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7e600\u003e: \n failed to create HCP cluster cluster-candidate-4-21-zb4zw7: failed waiting for cluster=\"cluster-candidate-4-21-zb4zw7\" in resourcegroup=\"rg-candidate-4-21-2n6qpzrjsvhw\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f39df5e-409a-4851-b372-be5717487b61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f39df5e-409a-4851-b372-be5717487b61\",\n \"name\": \"5f39df5e-409a-4851-b372-be5717487b61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:26.840054334Z\",\n \"endTime\": \"2026-08-25T14:41:34.377011529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-2n6qpzrjsvhw/cluster-candidate-4-21-zb4zw7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7e600\u003e: \n failed to create HCP cluster cluster-candidate-4-21-zb4zw7: failed waiting for cluster=\"cluster-candidate-4-21-zb4zw7\" in resourcegroup=\"rg-candidate-4-21-2n6qpzrjsvhw\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f39df5e-409a-4851-b372-be5717487b61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f39df5e-409a-4851-b372-be5717487b61\",\n \"name\": \"5f39df5e-409a-4851-b372-be5717487b61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:26.840054334Z\",\n \"endTime\": \"2026-08-25T14:41:34.377011529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d2f69825c0266b5c3b32966ab8eb4271f8df556438f9df09313698b88b85a6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"44ad071145b6440bebd0a2dc3f06efae0013a7f9125c8d5d38fc3bea9f263863","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:739]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c9840\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-wpzzs6\" in resourcegroup=\"rg-np-dg-minor-wpzzs6-6jql8zwjxxlb\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"name\": \"73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:50.588975119Z\",\n \"endTime\": \"2026-08-25T14:54:53.875483949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:739]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c9840\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-wpzzs6\" in resourcegroup=\"rg-np-dg-minor-wpzzs6-6jql8zwjxxlb\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"name\": \"73b2139d-e941-4022-8bf3-72abae50ef1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:50.588975119Z\",\n \"endTime\": \"2026-08-25T14:54:53.875483949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9456a3452150fe457953a0e81f6109e346504c9dc441eef591c5591845f15319","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"4a2ed9f21c01469e954cb46911f58a6903c7bef2bb37ab47e1b4136d17c074f9","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0ef80\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-bhfbpkdqqdrn\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0dda70b-836c-4894-a3d0-be51c86b05cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0dda70b-836c-4894-a3d0-be51c86b05cf\",\n \"name\": \"c0dda70b-836c-4894-a3d0-be51c86b05cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:44.500975Z\",\n \"endTime\": \"2026-08-25T14:41:44.271185619Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0ef80\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-bhfbpkdqqdrn\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0dda70b-836c-4894-a3d0-be51c86b05cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0dda70b-836c-4894-a3d0-be51c86b05cf\",\n \"name\": \"c0dda70b-836c-4894-a3d0-be51c86b05cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:44.500975Z\",\n \"endTime\": \"2026-08-25T14:41:44.271185619Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b2a206777e3c2e263eb73a65f0a54dceb5204fb4ac1b98cbc792414434741767","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"0df35959863c4a18d18241249045541a76f7fb86ff7bc0c329f8da11e7941ea4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-4z7lpz\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e04260\u003e: \n failed to create HCP cluster cluster-zstream-5-0-4z7lpz: failed waiting for cluster=\"cluster-zstream-5-0-4z7lpz\" in resourcegroup=\"rg-zstream-upgrade-5-0-4tgw65s225n8\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e8c47b5-1463-40b5-871f-63030d9c0472\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e8c47b5-1463-40b5-871f-63030d9c0472\",\n \"name\": \"8e8c47b5-1463-40b5-871f-63030d9c0472\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:33.195161833Z\",\n \"endTime\": \"2026-08-25T14:41:34.370521419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-4z7lpz\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e04260\u003e: \n failed to create HCP cluster cluster-zstream-5-0-4z7lpz: failed waiting for cluster=\"cluster-zstream-5-0-4z7lpz\" in resourcegroup=\"rg-zstream-upgrade-5-0-4tgw65s225n8\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e8c47b5-1463-40b5-871f-63030d9c0472\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e8c47b5-1463-40b5-871f-63030d9c0472\",\n \"name\": \"8e8c47b5-1463-40b5-871f-63030d9c0472\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:33.195161833Z\",\n \"endTime\": \"2026-08-25T14:41:34.370521419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c53cad041e31c5e1de335bbad808f0d6335d49bf42a532ec3fd0492f824e3327","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"e8bbe3fd18272cae3ce8b375d491cf16615e129bb95cc62f5766586a6be8de21","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-tzvdrp4rlnjq/cluster-candidate-4-20-txnzsg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f05420\u003e: \n failed to create HCP cluster cluster-candidate-4-20-txnzsg: failed waiting for cluster=\"cluster-candidate-4-20-txnzsg\" in resourcegroup=\"rg-candidate-4-20-tzvdrp4rlnjq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb25a348-e3b8-4c82-9665-cf01bc14f70f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb25a348-e3b8-4c82-9665-cf01bc14f70f\",\n \"name\": \"bb25a348-e3b8-4c82-9665-cf01bc14f70f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:26.331653993Z\",\n \"endTime\": \"2026-08-25T14:41:34.332024135Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-tzvdrp4rlnjq/cluster-candidate-4-20-txnzsg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f05420\u003e: \n failed to create HCP cluster cluster-candidate-4-20-txnzsg: failed waiting for cluster=\"cluster-candidate-4-20-txnzsg\" in resourcegroup=\"rg-candidate-4-20-tzvdrp4rlnjq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb25a348-e3b8-4c82-9665-cf01bc14f70f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb25a348-e3b8-4c82-9665-cf01bc14f70f\",\n \"name\": \"bb25a348-e3b8-4c82-9665-cf01bc14f70f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:26.331653993Z\",\n \"endTime\": \"2026-08-25T14:41:34.332024135Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d3ba666d5ddd9fe2f7d717b0ccd12de129419576dd545509f247364573754e64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"b217f98ca261620a2196fa549c1345074f0ad31609adefc4d0a95125d87e751a","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0afc0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gvltc2gr7wfl\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"name\": \"9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.827625738Z\",\n \"endTime\": \"2026-08-25T14:43:14.39497302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0afc0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gvltc2gr7wfl\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"name\": \"9a8b4b1e-639e-425c-8c76-d099661899e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.827625738Z\",\n \"endTime\": \"2026-08-25T14:43:14.39497302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dfa884eb706fd21d127bdde48d7270acca3a90fc880525457e6b8fc9109126e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"2f2e10c3f96af5d17d27eaefc2cdd8f45ad43cf97747092ff970d29300a33cb8","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code\nExpected\n \u003cstring\u003e: ...\nto contain substring\n \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code\nExpected\n \u003cstring\u003e: ...\nto contain substring\n \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5dd5ef6724beffab1f42a094be91cb72cdbac22ffe68c01aeeec9da825c19fa3","failure_pattern":"error for empty VM name should contain 400 status code","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code"}},{"row_id":"e3dc32af1fd5b57724bee991f15620764c0ad50049b542aeb0b4b3537c420a29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","failed_at":"e2e","signature_id":"dabbb7f9069ce3251099eefa51315c3d0e7f90c58175bd7bad32d9f2f8d81b40","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00196e760\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dsvpvl48ct65\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aaa93704-3b70-47e4-b524-6a5f42adba5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aaa93704-3b70-47e4-b524-6a5f42adba5d\",\n \"name\": \"aaa93704-3b70-47e4-b524-6a5f42adba5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:38.164192939Z\",\n \"endTime\": \"2026-08-25T14:41:44.371074479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00196e760\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dsvpvl48ct65\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aaa93704-3b70-47e4-b524-6a5f42adba5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aaa93704-3b70-47e4-b524-6a5f42adba5d\",\n \"name\": \"aaa93704-3b70-47e4-b524-6a5f42adba5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:38.164192939Z\",\n \"endTime\": \"2026-08-25T14:41:44.371074479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","5dd5ef6724beffab1f42a094be91cb72cdbac22ffe68c01aeeec9da825c19fa3","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:21:13Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"061b21f8d732d5d5291a44353a7e611fd2be69536b6b6058f8bfa3b6250d398a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"93381f18ce633ba4d082796c83975ebeab65d65a6f8d890e5eb3fce8495ce24b","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee4940\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-h62zblbg9qvp\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/708b2a4f-d727-42ba-851b-4c217c4509be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/708b2a4f-d727-42ba-851b-4c217c4509be\",\n \"name\": \"708b2a4f-d727-42ba-851b-4c217c4509be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:09.885838713Z\",\n \"endTime\": \"2026-08-25T14:40:13.678306778Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee4940\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-h62zblbg9qvp\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/708b2a4f-d727-42ba-851b-4c217c4509be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/708b2a4f-d727-42ba-851b-4c217c4509be\",\n \"name\": \"708b2a4f-d727-42ba-851b-4c217c4509be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:09.885838713Z\",\n \"endTime\": \"2026-08-25T14:40:13.678306778Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"06df0131b7eb55b7b86bbb3b4637afb7e2eee6a3061b1bce8d31fe675c1c3d27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"a9c57aba5f42be110377a5cf9ef9c173f8bee026a837d5411882de02fec7d3cc","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019daa40\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-g5pwkrb85fq6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"name\": \"a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:12.44361008Z\",\n \"endTime\": \"2026-08-25T14:40:13.632066828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019daa40\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-g5pwkrb85fq6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"name\": \"a089fdb6-b47d-4203-b58e-db6f9734d598\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:12.44361008Z\",\n \"endTime\": \"2026-08-25T14:40:13.632066828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2987053a15f7116a88e052c10fc06c88fe319cde80551c33d6a06b1e1de77ff9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"62e1bcec648cf6ca85b4778d6a217f22d6ca9de165a453fb96f3ba95214f5341","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019ee900\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bczkkzw624lz\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b1cb529-87ca-48d5-8037-5361bfdad717\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b1cb529-87ca-48d5-8037-5361bfdad717\",\n \"name\": \"0b1cb529-87ca-48d5-8037-5361bfdad717\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:32.618109568Z\",\n \"endTime\": \"2026-08-25T14:40:33.581291257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-network-operator, catalog-operator, olm-operator, ingress-operator, redhat-operators-catalog, packageserver, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019ee900\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bczkkzw624lz\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b1cb529-87ca-48d5-8037-5361bfdad717\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b1cb529-87ca-48d5-8037-5361bfdad717\",\n \"name\": \"0b1cb529-87ca-48d5-8037-5361bfdad717\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:32.618109568Z\",\n \"endTime\": \"2026-08-25T14:40:33.581291257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-network-operator, catalog-operator, olm-operator, ingress-operator, redhat-operators-catalog, packageserver, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"30490089ba9e6504c512b1df6d0861102a659cc4f2ede82fa2b1f6c7b971fbe7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"5ae8b49cfd62a1fd2be6d40eb9b8f7b2df783c47f66002e18ea549ef9fae6614","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00116a220\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-b9gsdkcb5wjn\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac2c2af3-451a-4ece-96a8-63d551193d0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac2c2af3-451a-4ece-96a8-63d551193d0d\",\n \"name\": \"ac2c2af3-451a-4ece-96a8-63d551193d0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:09.74317293Z\",\n \"endTime\": \"2026-08-25T14:40:13.66125404Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00116a220\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-b9gsdkcb5wjn\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac2c2af3-451a-4ece-96a8-63d551193d0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac2c2af3-451a-4ece-96a8-63d551193d0d\",\n \"name\": \"ac2c2af3-451a-4ece-96a8-63d551193d0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:09.74317293Z\",\n \"endTime\": \"2026-08-25T14:40:13.66125404Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"436346a5b8a9c39e04402a62d396ced584b806bddc793891b8ea08c238428a0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"3ee97bfc9952749cbcf036e0d8407c3873b852dade18372153b3dfc59818dac8","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:460]: failed to create nodepool nps-420-422 at version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e09e40\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-trdpwk\" in resourcegroup=\"rg-np-skip-minor-trdpwk-wpbsh8r8pf85\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4046da41-b84c-482b-b6f6-53a4043531fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4046da41-b84c-482b-b6f6-53a4043531fd\",\n \"name\": \"4046da41-b84c-482b-b6f6-53a4043531fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:48.059407214Z\",\n \"endTime\": \"2026-08-25T14:54:53.654727258Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:460]: failed to create nodepool nps-420-422 at version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e09e40\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-trdpwk\" in resourcegroup=\"rg-np-skip-minor-trdpwk-wpbsh8r8pf85\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4046da41-b84c-482b-b6f6-53a4043531fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4046da41-b84c-482b-b6f6-53a4043531fd\",\n \"name\": \"4046da41-b84c-482b-b6f6-53a4043531fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:35:48.059407214Z\",\n \"endTime\": \"2026-08-25T14:54:53.654727258Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b200011dd2cf6fd1b6ed577e31ecc9050b8c1dc013e61c884065f0b38d3d426","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"268457b3abac70cef4adc34ed78ed5e91defa790eb3f46b44d42d7bf24a4ab3d","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a5060\u003e: \n failed to create HCP cluster np-dg-minor-gzvdzt: failed waiting for cluster=\"np-dg-minor-gzvdzt\" in resourcegroup=\"rg-np-dg-minor-gzvdzt-zrllgdp7qt99\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\",\n \"name\": \"5b036348-ef9c-491d-be11-832d77741098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:00.551807818Z\",\n \"endTime\": \"2026-08-25T14:40:03.659175976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a5060\u003e: \n failed to create HCP cluster np-dg-minor-gzvdzt: failed waiting for cluster=\"np-dg-minor-gzvdzt\" in resourcegroup=\"rg-np-dg-minor-gzvdzt-zrllgdp7qt99\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\",\n \"name\": \"5b036348-ef9c-491d-be11-832d77741098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:00.551807818Z\",\n \"endTime\": \"2026-08-25T14:40:03.659175976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"daeefc59f58572b03a630f3ab57fcec5e15bddea741d2154444f66cb96ddf3d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"55b4c8ce3f2fbb701649fcb4f8bdc4be8e1d895be3b9e63909a1b911f7ad05f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"489782a342d805c0338dec6f638d02703430b0e8f541b6a5142fbe16ac3c7ae8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-2znfmbcxp8tg/cluster-candidate-4-22-kbnpqt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff0200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kbnpqt: failed waiting for cluster=\"cluster-candidate-4-22-kbnpqt\" in resourcegroup=\"rg-candidate-4-22-2znfmbcxp8tg\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f24431ca-76c6-43bb-b4ca-aaa1fba71d4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f24431ca-76c6-43bb-b4ca-aaa1fba71d4b\",\n \"name\": \"f24431ca-76c6-43bb-b4ca-aaa1fba71d4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:12.456614245Z\",\n \"endTime\": \"2026-08-25T14:40:13.674948976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-2znfmbcxp8tg/cluster-candidate-4-22-kbnpqt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff0200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kbnpqt: failed waiting for cluster=\"cluster-candidate-4-22-kbnpqt\" in resourcegroup=\"rg-candidate-4-22-2znfmbcxp8tg\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f24431ca-76c6-43bb-b4ca-aaa1fba71d4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f24431ca-76c6-43bb-b4ca-aaa1fba71d4b\",\n \"name\": \"f24431ca-76c6-43bb-b4ca-aaa1fba71d4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:21:12.456614245Z\",\n \"endTime\": \"2026-08-25T14:40:13.674948976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"679848afcee559e6914d4d5a6ca6f9c41218c0d182733341258ceeb88e952123","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"6cd3b4ec3a78c5076ca55dacbf68ec344e1698246fdf7d2fed60e6b4e6630276","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--v8ffjmksbzdl/cluster-hs-pre-8jd46w should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001938ac0\u003e: \n failed to create HCP cluster cluster-hs-pre-8jd46w: failed waiting for cluster=\"cluster-hs-pre-8jd46w\" in resourcegroup=\"rg-hs-pre--v8ffjmksbzdl\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/267a7d0a-658c-4265-8e01-7307d8e5f38b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/267a7d0a-658c-4265-8e01-7307d8e5f38b\",\n \"name\": \"267a7d0a-658c-4265-8e01-7307d8e5f38b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:20:39.09953391Z\",\n \"endTime\": \"2026-08-25T14:39:43.525617914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--v8ffjmksbzdl/cluster-hs-pre-8jd46w should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001938ac0\u003e: \n failed to create HCP cluster cluster-hs-pre-8jd46w: failed waiting for cluster=\"cluster-hs-pre-8jd46w\" in resourcegroup=\"rg-hs-pre--v8ffjmksbzdl\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/267a7d0a-658c-4265-8e01-7307d8e5f38b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/267a7d0a-658c-4265-8e01-7307d8e5f38b\",\n \"name\": \"267a7d0a-658c-4265-8e01-7307d8e5f38b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:20:39.09953391Z\",\n \"endTime\": \"2026-08-25T14:39:43.525617914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e227199d49e7c520947bdf4809cc77c6d5af51ec238e61792e06d8113b1a3c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"2ee84b2de684d6bebbf318c8d5de7d1297cf8cffb2cae88ad176fe02897a3e46","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da2d80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-x6vhg5f425wn\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85b408c1-3b72-4317-9f37-da38ef217efb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85b408c1-3b72-4317-9f37-da38ef217efb\",\n \"name\": \"85b408c1-3b72-4317-9f37-da38ef217efb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:37:38.068129685Z\",\n \"endTime\": \"2026-08-25T14:56:43.524475606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da2d80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-x6vhg5f425wn\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85b408c1-3b72-4317-9f37-da38ef217efb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85b408c1-3b72-4317-9f37-da38ef217efb\",\n \"name\": \"85b408c1-3b72-4317-9f37-da38ef217efb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:37:38.068129685Z\",\n \"endTime\": \"2026-08-25T14:56:43.524475606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a68cb065ecd813b156536889b151a03f1df1db8d6e06dcdb69e64110058d0ca7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"51019fbbc87d86ea23801cb65819ccf80ec820166053874e8a6fcf0732993915","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002200620\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2pmrbkkbtqsr\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"name\": \"27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:13.29802062Z\",\n \"endTime\": \"2026-08-25T14:42:13.632929999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, cluster-version-operator, cluster-storage-operator, certified-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, redhat-operators-catalog, openshift-apiserver, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, olm-operator, community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, ingress-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002200620\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2pmrbkkbtqsr\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"name\": \"27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:13.29802062Z\",\n \"endTime\": \"2026-08-25T14:42:13.632929999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, cluster-version-operator, cluster-storage-operator, certified-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, redhat-operators-catalog, openshift-apiserver, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, olm-operator, community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, ingress-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fc644a81a7601796e8c7e94724517f0056e9382ba0920b2aaa0529c449d6cba7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c3cdc7272282bb5e0a270d9568cb95044abf4fcf22ed2dd2804346837b03a720","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"f0717dee0439ce4b9da20978f82a9a18a9e1f1a19a95e531972424323361a844","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e26e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf47q9x8cxz7\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"name\": \"812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:21.177808774Z\",\n \"endTime\": \"2026-08-25T14:41:23.66756422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e26e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf47q9x8cxz7\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"name\": \"812a5a7d-5729-4953-b916-1cbe8aba438f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:21.177808774Z\",\n \"endTime\": \"2026-08-25T14:41:23.66756422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f2c9b6c3cc893a1112abe17bd6828d4b4d94610761399d33a7b637ed778c42e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","failed_at":"e2e","signature_id":"3bb6f89d65a0e5459009870c86357e81a73e7068ce6cf13f3fa374106ca56368","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fl7ppg8w76k6/cluster-candidate-5-0-9h5fbf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013b0300\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9h5fbf: failed waiting for cluster=\"cluster-candidate-5-0-9h5fbf\" in resourcegroup=\"rg-candidate-5-0-fl7ppg8w76k6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"name\": \"3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:20:49.919147854Z\",\n \"endTime\": \"2026-08-25T14:39:53.656202045Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator, dns-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fl7ppg8w76k6/cluster-candidate-5-0-9h5fbf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013b0300\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9h5fbf: failed waiting for cluster=\"cluster-candidate-5-0-9h5fbf\" in resourcegroup=\"rg-candidate-5-0-fl7ppg8w76k6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"name\": \"3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:20:49.919147854Z\",\n \"endTime\": \"2026-08-25T14:39:53.656202045Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator, dns-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1264ce1f65fb1c4e4b03ef0f0b493117b1b9becb99a5e03d914e5ae33ea84790","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["1264ce1f65fb1c4e4b03ef0f0b493117b1b9becb99a5e03d914e5ae33ea84790","39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","daeefc59f58572b03a630f3ab57fcec5e15bddea741d2154444f66cb96ddf3d9","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","fc644a81a7601796e8c7e94724517f0056e9382ba0920b2aaa0529c449d6cba7"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6641,"pr_state":"open","pr_sha":"2a2bf6c0e8f39a611bcc3192554e764493be1383","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:20:40Z"},"failed_at":["alert","e2e"],"failed_test_count":23,"occurrences":[{"row_id":"019eef111a9f7b52c6c9fdaf8430714769f058b7737fd00b4fcbf96ba675466b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"ebe5c6af0803fc1a7dab9b5770745552c68456e64c75e2851d5fb42f7169307d","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3c8e0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4z5qknlhfz6j\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"name\": \"00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.653354515Z\",\n \"endTime\": \"2026-08-25T14:42:33.765546046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3c8e0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4z5qknlhfz6j\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"name\": \"00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.653354515Z\",\n \"endTime\": \"2026-08-25T14:42:33.765546046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"efea2dcd245af6f421d5d48a15159d0a97c6fcbf251bcc721e928eae3008a031","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"09ef9296162dc8752f92c0fcce0a1beb3de26978796760cb2412a1f85c3cfbb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"86c42132d5d5019c3e30fa7d9fcde1ed01362afa33a0740202359c037a9d6aba","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e099e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-wdfkhlwcpc48\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3126ea30-d507-4147-af14-e885160a5806\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3126ea30-d507-4147-af14-e885160a5806\",\n \"name\": \"3126ea30-d507-4147-af14-e885160a5806\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.35552824Z\",\n \"endTime\": \"2026-08-25T14:43:13.692702059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, olm-operator, redhat-operators-catalog, packageserver, cluster-storage-operator, cluster-image-registry-operator, hosted-cluster-config-operator, ingress-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e099e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-wdfkhlwcpc48\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3126ea30-d507-4147-af14-e885160a5806\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3126ea30-d507-4147-af14-e885160a5806\",\n \"name\": \"3126ea30-d507-4147-af14-e885160a5806\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:12.35552824Z\",\n \"endTime\": \"2026-08-25T14:43:13.692702059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, olm-operator, redhat-operators-catalog, packageserver, cluster-storage-operator, cluster-image-registry-operator, hosted-cluster-config-operator, ingress-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ae0e34cfdeace9ac747e097fe852d1e02661929b69c494a728e8de28cee5e3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"9fed5947fac9eb6a4be4cefb16ab0947113474d3eee1eb30b287e0d183316002","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-28m5d4v9fkbw/cluster-candidate-4-22-ng5jqz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8a8a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ng5jqz: failed waiting for cluster=\"cluster-candidate-4-22-ng5jqz\" in resourcegroup=\"rg-candidate-4-22-28m5d4v9fkbw\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bd2042e-01bd-41c0-98fb-a47ff3536356\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bd2042e-01bd-41c0-98fb-a47ff3536356\",\n \"name\": \"9bd2042e-01bd-41c0-98fb-a47ff3536356\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:58.928780883Z\",\n \"endTime\": \"2026-08-25T14:42:03.744023943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-28m5d4v9fkbw/cluster-candidate-4-22-ng5jqz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8a8a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ng5jqz: failed waiting for cluster=\"cluster-candidate-4-22-ng5jqz\" in resourcegroup=\"rg-candidate-4-22-28m5d4v9fkbw\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bd2042e-01bd-41c0-98fb-a47ff3536356\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bd2042e-01bd-41c0-98fb-a47ff3536356\",\n \"name\": \"9bd2042e-01bd-41c0-98fb-a47ff3536356\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:22:58.928780883Z\",\n \"endTime\": \"2026-08-25T14:42:03.744023943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16fb43fa6c3b4ee4e49d484415e46b1c58a5a0ae2be09d1529f6d8e4b1afc19a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"de16d3bee3a53f0651e2d553859d03cd36da7aee381249015bf8db424e02b9bb","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e88e20\u003e: \n failed to create HCP cluster ea-listhfgh7b: failed waiting for cluster=\"ea-listhfgh7b\" in resourcegroup=\"ea-list-6p4sr9k5q7bf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\",\n \"name\": \"62436e44-f35d-4583-a471-e3b8c1476144\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:30.392692104Z\",\n \"endTime\": \"2026-08-25T14:42:33.727154801Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e88e20\u003e: \n failed to create HCP cluster ea-listhfgh7b: failed waiting for cluster=\"ea-listhfgh7b\" in resourcegroup=\"ea-list-6p4sr9k5q7bf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\",\n \"name\": \"62436e44-f35d-4583-a471-e3b8c1476144\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:30.392692104Z\",\n \"endTime\": \"2026-08-25T14:42:33.727154801Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"18d6e2a9cb260b31d545311b1535143ff9104b7e5ebc68ce43da46da20994032","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"3c8445447816fab0494fb15002cb32f86c4fc7cc3167f611c626f8d9af1ff88e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008f8400\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5f7fqf8wr7rh\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/996552c5-cfd8-458f-bb7e-b012f72bb6fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/996552c5-cfd8-458f-bb7e-b012f72bb6fd\",\n \"name\": \"996552c5-cfd8-458f-bb7e-b012f72bb6fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.305316303Z\",\n \"endTime\": \"2026-08-25T14:42:23.727933134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008f8400\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5f7fqf8wr7rh\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/996552c5-cfd8-458f-bb7e-b012f72bb6fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/996552c5-cfd8-458f-bb7e-b012f72bb6fd\",\n \"name\": \"996552c5-cfd8-458f-bb7e-b012f72bb6fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.305316303Z\",\n \"endTime\": \"2026-08-25T14:42:23.727933134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"19c82a265388d70cc7d32e3327f2de400afb61f667349cf2193f82ef69866a32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"31be970aed5c1350ca97847fcd67fc79e890462b95ed0fd2c70eb16c601cc4af","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-jfwr5v with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001072ac0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-jfwr5v: failed waiting for cluster=\"np-version-upgrade-cluster-jfwr5v\" in resourcegroup=\"rg-np-version-upgrade-jfwr5v-5pf2tksxbxvk\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f9c0f4c-5e66-444f-99cf-8a9d11c913f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f9c0f4c-5e66-444f-99cf-8a9d11c913f9\",\n \"name\": \"3f9c0f4c-5e66-444f-99cf-8a9d11c913f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:31.574965707Z\",\n \"endTime\": \"2026-08-25T14:42:33.731961599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-jfwr5v with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001072ac0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-jfwr5v: failed waiting for cluster=\"np-version-upgrade-cluster-jfwr5v\" in resourcegroup=\"rg-np-version-upgrade-jfwr5v-5pf2tksxbxvk\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f9c0f4c-5e66-444f-99cf-8a9d11c913f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f9c0f4c-5e66-444f-99cf-8a9d11c913f9\",\n \"name\": \"3f9c0f4c-5e66-444f-99cf-8a9d11c913f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:31.574965707Z\",\n \"endTime\": \"2026-08-25T14:42:33.731961599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"453aa104a09d7f981e9bbcf9bdcce060eb713acb048e848d34b947c8eee06c78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"7d1767d3e893f8c308c1422c3d2aa396037e232ca87a62275c5cabd76581cee7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-l7z8bjhz25gx/cluster-candidate-4-21-mj7vzs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000690a00\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mj7vzs: failed waiting for cluster=\"cluster-candidate-4-21-mj7vzs\" in resourcegroup=\"rg-candidate-4-21-l7z8bjhz25gx\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"name\": \"6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.273281186Z\",\n \"endTime\": \"2026-08-25T14:42:23.718787141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-l7z8bjhz25gx/cluster-candidate-4-21-mj7vzs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000690a00\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mj7vzs: failed waiting for cluster=\"cluster-candidate-4-21-mj7vzs\" in resourcegroup=\"rg-candidate-4-21-l7z8bjhz25gx\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"name\": \"6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.273281186Z\",\n \"endTime\": \"2026-08-25T14:42:23.718787141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52b927fb75e1a0e28c1c3ab5099caef3882ee435ade1e450f06515148c6a9d46","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d13fe09cead55eb82e34fe665d8412799eb24ff92e12c9ef4bf3682d629c3f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"3f0f7b3d38506c573a5570dd4fc273d4974d9a5328b2080e03b587e59e9463aa","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c43c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-zcf2d6kk52j7\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/208d78c8-6efe-4c95-b106-65d2487b4d6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/208d78c8-6efe-4c95-b106-65d2487b4d6f\",\n \"name\": \"208d78c8-6efe-4c95-b106-65d2487b4d6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:10.560757862Z\",\n \"endTime\": \"2026-08-25T14:42:13.708090431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c43c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-zcf2d6kk52j7\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/208d78c8-6efe-4c95-b106-65d2487b4d6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/208d78c8-6efe-4c95-b106-65d2487b4d6f\",\n \"name\": \"208d78c8-6efe-4c95-b106-65d2487b4d6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:10.560757862Z\",\n \"endTime\": \"2026-08-25T14:42:13.708090431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6201840179105a6445a63587fc9679d28440ab60ccfd71608ba914e87314becb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"993ce51c1d7313554e2033361bc0c241e0479977c567c64410226f52de9d8006","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ca240\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-d4mb4xht7542\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"name\": \"d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.491781901Z\",\n \"endTime\": \"2026-08-25T14:42:33.748439526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-policy-controller, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, machine-approver, openshift-route-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, packageserver, openshift-controller-manager, dns-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ca240\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-d4mb4xht7542\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"name\": \"d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:29.491781901Z\",\n \"endTime\": \"2026-08-25T14:42:33.748439526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-policy-controller, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, machine-approver, openshift-route-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, packageserver, openshift-controller-manager, dns-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fc644a81a7601796e8c7e94724517f0056e9382ba0920b2aaa0529c449d6cba7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"803df07ab2626e9d76876fe8264002851fb74af6011be93bc87ee8bf36ea6957","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"50cfe241bd776c63ead0185c89db291d20bfefd14fa4462a1eff23a8bb7280f2","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4a7e0\u003e: \n failed to create HCP cluster np-dg-minor-9qfj96: failed waiting for cluster=\"np-dg-minor-9qfj96\" in resourcegroup=\"rg-np-dg-minor-9qfj96-vfn2xzp2hh4x\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8ce4f4e-f284-4530-a975-e577f84037f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8ce4f4e-f284-4530-a975-e577f84037f8\",\n \"name\": \"c8ce4f4e-f284-4530-a975-e577f84037f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:10.502180324Z\",\n \"endTime\": \"2026-08-25T14:42:13.711516452Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4a7e0\u003e: \n failed to create HCP cluster np-dg-minor-9qfj96: failed waiting for cluster=\"np-dg-minor-9qfj96\" in resourcegroup=\"rg-np-dg-minor-9qfj96-vfn2xzp2hh4x\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8ce4f4e-f284-4530-a975-e577f84037f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8ce4f4e-f284-4530-a975-e577f84037f8\",\n \"name\": \"c8ce4f4e-f284-4530-a975-e577f84037f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:10.502180324Z\",\n \"endTime\": \"2026-08-25T14:42:13.711516452Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"872557275661dd0428b695940fd600f1198346fa5a3cd91f09be27ca8534a378","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"8d522cd2839df39682d21c0af249f6118f11366a2bf105536db9eb68cb5a9451","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e6ae0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vrnlz7rb6wl4\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f07a9ee-2395-4d42-aa33-38519e85c49d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f07a9ee-2395-4d42-aa33-38519e85c49d\",\n \"name\": \"8f07a9ee-2395-4d42-aa33-38519e85c49d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:37:18.839720203Z\",\n \"endTime\": \"2026-08-25T14:56:23.617637004Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e6ae0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vrnlz7rb6wl4\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f07a9ee-2395-4d42-aa33-38519e85c49d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f07a9ee-2395-4d42-aa33-38519e85c49d\",\n \"name\": \"8f07a9ee-2395-4d42-aa33-38519e85c49d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:37:18.839720203Z\",\n \"endTime\": \"2026-08-25T14:56:23.617637004Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8eed8f92233e82a54b1589c3e38e66b40d1fed55e6a1b1996f4ce276361ff15f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"f04cb5f20e7c296508246409d131f2f25950ae8cf510b0c95d54eb7e1af46d29","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6640\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-97gvpsltv9sn\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"name\": \"55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:51.992542762Z\",\n \"endTime\": \"2026-08-25T14:43:53.990674833Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6640\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-97gvpsltv9sn\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"name\": \"55ca21a5-264c-4b89-812c-1bd654f8f905\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:24:51.992542762Z\",\n \"endTime\": \"2026-08-25T14:43:53.990674833Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56aad0b4262c7cedf5722ba39358ba8c22af49e4bbcaade93736c9c50ce6eb1a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b789d4309e4cf4f7120b24fc36b44a4e4ed3523c49eb3313cf16d890623b166","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"8264131fc2b45d2f5ca5d9936dd76c0aeb91f30f562df92f37bbc56a238e6d26","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a922c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-4m6h5kd6h7vt\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acbfe6bf-3ab4-49b1-a7fb-9d29f2abc4ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acbfe6bf-3ab4-49b1-a7fb-9d29f2abc4ed\",\n \"name\": \"acbfe6bf-3ab4-49b1-a7fb-9d29f2abc4ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.115641816Z\",\n \"endTime\": \"2026-08-25T14:42:13.649502526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a922c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-4m6h5kd6h7vt\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acbfe6bf-3ab4-49b1-a7fb-9d29f2abc4ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acbfe6bf-3ab4-49b1-a7fb-9d29f2abc4ed\",\n \"name\": \"acbfe6bf-3ab4-49b1-a7fb-9d29f2abc4ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.115641816Z\",\n \"endTime\": \"2026-08-25T14:42:13.649502526Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b15f32166edb18146ef11aad2b9f7a93fd1e7a0387d068783840bb268f22caa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"alert","signature_id":"1ae78d946fc05772983eafd321d0a8f04e93c242870b0eb8a31923e9082da049","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 58 time(s) (19 unknown, 39 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T14:37:24Z\n Ended: 2026-08-25T14:50:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlegoo9bd727bjcnuetqvupvv32qu8-p0k5e8q6m7g0x4g\", pod=\"router-6d67c7888f-hqj2v\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlegoo9bd727bjcnuetqvupvv32qu8-p0k5e8q6m7g0x4g/router-6d67c7888f-hqj2v has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T14:37:24Z\n Ended: 2026-08-25T14:43:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlejejkoqkg3lci2l992u8dnigrvj2-p9y5g4g2q1m8z4y\", pod=\"router-cf94d77fb-gjndj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlejejkoqkg3lci2l992u8dnigrvj2-p9y5g4g2q1m8z4y/router-cf94d77fb-gjndj has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T14:37:24Z\n Ended: 2026-08-25T14:50:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlegoo9bd727bjcnuetqvupvv32qu8-p0k5e8q6m7g0x4g\", pod=\"router-6d67c7888f-9jf9z\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlegoo9bd727bjcnuetqvupvv32qu8-p0k5e8q6m7g0x4g/router-6d67c7888f-9jf9z has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:45:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlejsuuod9gocuvio5n9l642vqd4eh-x6h6n4p7y9s8w9d\", pod=\"dns-operator-6c6b4fb7cd-pfdnm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlejsuuod9gocuvio5n9l642vqd4eh-x6h6n4p7y9s8w9d/dns-operator-6c6b4fb7cd-pfdnm has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlem78mrdia1hmshmk1is9cukdlrjd-t6l2v9o7c8z1u3c\", pod=\"router-5667b96f5d-42xln\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlem78mrdia1hmshmk1is9cukdlrjd-t6l2v9o7c8z1u3c/router-5667b96f5d-42xln has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:45:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlejsuuod9gocuvio5n9l642vqd4eh-x6h6n4p7y9s8w9d\", pod=\"cluster-image-registry-operator-5656f84f95-g7lnk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlejsuuod9gocuvio5n9l642vqd4eh-x6h6n4p7y9s8w9d/cluster-image-registry-operator-5656f84f95-g7lnk has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:45:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlfd1jcj2cm3mnt32pp6a788r14tqr-a3w9p7o0b2m6y7d\", pod=\"router-687b49f995-ztm4t\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlfd1jcj2cm3mnt32pp6a788r14tqr-a3w9p7o0b2m6y7d/router-687b49f995-ztm4t has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlem6anaqvl6nuptvbhspigoa8pi7f-q7i5x4f3a4u8u4x\", pod=\"router-c5b76f6b7-vbmjg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlem6anaqvl6nuptvbhspigoa8pi7f-q7i5x4f3a4u8u4x/router-c5b76f6b7-vbmjg has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:46:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlejsuuod9gocuvio5n9l642vqd4eh-x6h6n4p7y9s8w9d\", pod=\"ingress-operator-687f786bc9-67mkc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlejsuuod9gocuvio5n9l642vqd4eh-x6h6n4p7y9s8w9d/ingress-operator-687f786bc9-67mkc has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:45:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdleopo6msip9htu3nm9s74075vsivg-f2a3p1v9l8c5q7c\", pod=\"dns-operator-5655b66c99-ltgth\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdleopo6msip9htu3nm9s74075vsivg-f2a3p1v9l8c5q7c/dns-operator-5655b66c99-ltgth has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:45:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdleopo6msip9htu3nm9s74075vsivg-f2a3p1v9l8c5q7c\", pod=\"cluster-image-registry-operator-6b6c7779fc-7ws99\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdleopo6msip9htu3nm9s74075vsivg-f2a3p1v9l8c5q7c/cluster-image-registry-operator-6b6c7779fc-7ws99 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:45:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdleopo6msip9htu3nm9s74075vsivg-f2a3p1v9l8c5q7c\", pod=\"ingress-operator-6bd6477fcc-lxkk4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdleopo6msip9htu3nm9s74075vsivg-f2a3p1v9l8c5q7c/ingress-operator-6bd6477fcc-lxkk4 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T14:39:24Z\n Ended: 2026-08-25T14:47:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdleolvj8jkpoukleh4otfco23bj7gk-v20260901\", pod=\"router-5f7bcbb5c8-hqd9h\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdleolvj8jkpoukleh4otfco23bj7gk-v20260901/router-5f7bcbb5c8-hqd9h has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T14:40:25Z\n Ended: 2026-08-25T14:52:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlf3e90766j6dqef5ebgosi6grm0nf-private-kas\", pod=\"router-645d7f96c4-m7mfv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlf3e90766j6dqef5ebgosi6grm0nf-private-kas/router-645d7f96c4-m7mfv has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T14:41:24Z\n Ended: 2026-08-25T14:48:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlem6anaqvl6nuptvbhspigoa8pi7f-q7i5x4f3a4u8u4x\", pod=\"ingress-operator-5dbd9fdbb6-lk864\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlem6anaqvl6nuptvbhspigoa8pi7f-q7i5x4f3a4u8u4x/ingress-operator-5dbd9fdbb6-lk864 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T14:41:25Z\n Ended: 2026-08-25T14:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlem6anaqvl6nuptvbhspigoa8pi7f-q7i5x4f3a4u8u4x\", pod=\"cluster-network-operator-7c7f7f8b99-5kwm5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlem6anaqvl6nuptvbhspigoa8pi7f-q7i5x4f3a4u8u4x/cluster-network-operator-7c7f7f8b99-5kwm5 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T14:41:25Z\n Ended: 2026-08-25T14:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlem4jmsj57unhcuuumlc6abnimmop-v7e8k2i7y9u0b2p\", pod=\"router-58d8d6898f-k9nqv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlem4jmsj57unhcuuumlc6abnimmop-v7e8k2i7y9u0b2p/router-58d8d6898f-k9nqv has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T14:41:25Z\n Ended: 2026-08-25T14:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdleop0e81vroks1mrdm6r5jl79961q-q1n9m0x8z5a8w9z\", pod=\"router-5c8d77f9c7-wxfgv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdleop0e81vroks1mrdm6r5jl79961q-q1n9m0x8z5a8w9z/router-5c8d77f9c7-wxfgv has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T14:42:24Z\n Ended: 2026-08-25T14:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9549056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdlem3cfjc7gd3ldptserbelahoc1dp-f9m3o7e8f8j6d7j\", pod=\"router-6d8949db64-nmdv8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdlem3cfjc7gd3ldptserbelahoc1dp-f9m3o7e8f8j6d7j/router-6d8949db64-nmdv8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c072f191734f4865f9df032977153364c62264a844aa60ce6b7d7f7bcd032195","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"1f29965bd436288d269b00ceb20f18dc47e327d03580c8db54436fbf801d2664","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009158e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-stcdz6kncxhr\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/567d6c13-28cc-4446-ab15-21bd709a116c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/567d6c13-28cc-4446-ab15-21bd709a116c\",\n \"name\": \"567d6c13-28cc-4446-ab15-21bd709a116c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:08.741636847Z\",\n \"endTime\": \"2026-08-25T14:42:13.719295306Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009158e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-stcdz6kncxhr\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/567d6c13-28cc-4446-ab15-21bd709a116c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/567d6c13-28cc-4446-ab15-21bd709a116c\",\n \"name\": \"567d6c13-28cc-4446-ab15-21bd709a116c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:08.741636847Z\",\n \"endTime\": \"2026-08-25T14:42:13.719295306Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc81fa858d4142a8c990147dcd438ede101d9bf20802e348784460a6100a54cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"eed43f59ecea085b1ffeb037fb6bd3536093190e7881447ceae828484fef5878","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011468e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w7786nk5nznp\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"name\": \"9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.963379665Z\",\n \"endTime\": \"2026-08-25T14:42:23.660042904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011468e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w7786nk5nznp\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"name\": \"9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.963379665Z\",\n \"endTime\": \"2026-08-25T14:42:23.660042904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"efea2dcd245af6f421d5d48a15159d0a97c6fcbf251bcc721e928eae3008a031","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d2e2f37600ac0aafb854c6a67effa0ad376ec21e7fbc6de0a40c011c345a872a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"01b87695dd9b2116a68fd8b6e0c91abd3e780e11edb0177bfb654801580cf72d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fsbfnssnhqkf/cluster-candidate-5-0-2knmvc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e46260\u003e: \n failed to create HCP cluster cluster-candidate-5-0-2knmvc: failed waiting for cluster=\"cluster-candidate-5-0-2knmvc\" in resourcegroup=\"rg-candidate-5-0-fsbfnssnhqkf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc01a940-8988-427e-9d74-c468fbaf0fee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc01a940-8988-427e-9d74-c468fbaf0fee\",\n \"name\": \"cc01a940-8988-427e-9d74-c468fbaf0fee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.159652548Z\",\n \"endTime\": \"2026-08-25T14:42:23.741564783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fsbfnssnhqkf/cluster-candidate-5-0-2knmvc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e46260\u003e: \n failed to create HCP cluster cluster-candidate-5-0-2knmvc: failed waiting for cluster=\"cluster-candidate-5-0-2knmvc\" in resourcegroup=\"rg-candidate-5-0-fsbfnssnhqkf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc01a940-8988-427e-9d74-c468fbaf0fee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc01a940-8988-427e-9d74-c468fbaf0fee\",\n \"name\": \"cc01a940-8988-427e-9d74-c468fbaf0fee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.159652548Z\",\n \"endTime\": \"2026-08-25T14:42:23.741564783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d4a0f5a5756760cd3b5702a1264e82dac32d51feaf3df60af086179bb3a1ca71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"69a7645ef69f7c63a48191f6533d04ca0a326a56df092b3558b2f664fafc5a80","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14a20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ld6hfp9nvt27\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df760612-adee-4fe6-b2cc-01751b699824\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df760612-adee-4fe6-b2cc-01751b699824\",\n \"name\": \"df760612-adee-4fe6-b2cc-01751b699824\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:30.466260682Z\",\n \"endTime\": \"2026-08-25T14:42:33.758671656Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, openshift-route-controller-manager, cluster-storage-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-image-registry-operator, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, redhat-operators-catalog, packageserver, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14a20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ld6hfp9nvt27\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df760612-adee-4fe6-b2cc-01751b699824\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df760612-adee-4fe6-b2cc-01751b699824\",\n \"name\": \"df760612-adee-4fe6-b2cc-01751b699824\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:30.466260682Z\",\n \"endTime\": \"2026-08-25T14:42:33.758671656Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, openshift-route-controller-manager, cluster-storage-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-image-registry-operator, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, redhat-operators-catalog, packageserver, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"db9e8bbf71a4f4fcc097adf32bda28b4ecda8f77cfd1f121acfc8ffbc097430e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"alert","signature_id":"38c81dc69b06278ed233e0939f35e8c71cf8eff2f9cb495db24d74f240d5e1a1","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T15:14:00Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j9549056-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9549056/providers/microsoft.monitor/accounts/services-j9549056\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"e49b2707d2cd675b8eb92da53fe286c4a5e3bff02d220d7180b480e41e780ec7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"26cd31b13da1cd4b784cc2fc7816f909f73ac2fec28631501c0c3e5e8645ce51","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-qclt2j\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-qclt2j: failed waiting for cluster=\"cp-ystream-upgrade-4-21-qclt2j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2mbkqtdbwbbj\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"name\": \"c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.836508695Z\",\n \"endTime\": \"2026-08-25T14:42:13.731155696Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, dns-operator, csi-snapshot-controller-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-qclt2j\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-qclt2j: failed waiting for cluster=\"cp-ystream-upgrade-4-21-qclt2j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2mbkqtdbwbbj\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"name\": \"c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:09.836508695Z\",\n \"endTime\": \"2026-08-25T14:42:13.731155696Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, dns-operator, csi-snapshot-controller-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5a894851c2bcc53bc077fbbd00edddef3f1b3dd945f85daa36048e962b5272c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"09a91b6d59e4fbee757881e9171ec763df5844bb16934da2b1be95a803e452f3","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbcde0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ckzdqmgrqb4b\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c26091f-a379-4334-aa6c-e0a5fcf1a616\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c26091f-a379-4334-aa6c-e0a5fcf1a616\",\n \"name\": \"9c26091f-a379-4334-aa6c-e0a5fcf1a616\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:28.457957025Z\",\n \"endTime\": \"2026-08-25T14:44:33.709034069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbcde0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ckzdqmgrqb4b\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c26091f-a379-4334-aa6c-e0a5fcf1a616\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c26091f-a379-4334-aa6c-e0a5fcf1a616\",\n \"name\": \"9c26091f-a379-4334-aa6c-e0a5fcf1a616\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:25:28.457957025Z\",\n \"endTime\": \"2026-08-25T14:44:33.709034069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fde135189870bbd207c30af873a106ab6c965cc22b9db6151893feb07f845b78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"327150b988523757d3380ba7ee58ec97de40f9671944b2888f7d5817ed0ac98a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cgzvsfdzkmd6/cluster-candidate-4-20-gm7v89 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00146c980\u003e: \n failed to create HCP cluster cluster-candidate-4-20-gm7v89: failed waiting for cluster=\"cluster-candidate-4-20-gm7v89\" in resourcegroup=\"rg-candidate-4-20-cgzvsfdzkmd6\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"name\": \"048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.870414032Z\",\n \"endTime\": \"2026-08-25T14:42:23.711491128Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, dns-operator, hosted-cluster-config-operator, packageserver, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cgzvsfdzkmd6/cluster-candidate-4-20-gm7v89 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00146c980\u003e: \n failed to create HCP cluster cluster-candidate-4-20-gm7v89: failed waiting for cluster=\"cluster-candidate-4-20-gm7v89\" in resourcegroup=\"rg-candidate-4-20-cgzvsfdzkmd6\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"name\": \"048ffc99-876d-41b4-8fb3-df2166c912e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.870414032Z\",\n \"endTime\": \"2026-08-25T14:42:23.711491128Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, dns-operator, hosted-cluster-config-operator, packageserver, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6bc6e1fed29949cb7f899cfde9a499bb81c4498d10956c4f31ee3021f08a1b5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff010e8e20c5ba6c55f4ce025da60f2db3e43c045c5fb3725f330bbb7902a821","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","failed_at":"e2e","signature_id":"f096e2894e5607754bc52d3997abbbaf63ee7b84458d5fadbc6b046423af77c3","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kcshtglk78v4/cluster-hs-pre-j976nz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001828400\u003e: \n failed to create HCP cluster cluster-hs-pre-j976nz: failed waiting for cluster=\"cluster-hs-pre-j976nz\" in resourcegroup=\"rg-hs-pre--kcshtglk78v4\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/43093e9e-e93e-48ed-9142-17883753654a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/43093e9e-e93e-48ed-9142-17883753654a\",\n \"name\": \"43093e9e-e93e-48ed-9142-17883753654a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.671981847Z\",\n \"endTime\": \"2026-08-25T14:42:23.717184086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kcshtglk78v4/cluster-hs-pre-j976nz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001828400\u003e: \n failed to create HCP cluster cluster-hs-pre-j976nz: failed waiting for cluster=\"cluster-hs-pre-j976nz\" in resourcegroup=\"rg-hs-pre--kcshtglk78v4\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/43093e9e-e93e-48ed-9142-17883753654a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/43093e9e-e93e-48ed-9142-17883753654a\",\n \"name\": \"43093e9e-e93e-48ed-9142-17883753654a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:23:19.671981847Z\",\n \"endTime\": \"2026-08-25T14:42:23.717184086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":23,"failure_pattern_ids":["13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","52b927fb75e1a0e28c1c3ab5099caef3882ee435ade1e450f06515148c6a9d46","56aad0b4262c7cedf5722ba39358ba8c22af49e4bbcaade93736c9c50ce6eb1a","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","d6bc6e1fed29949cb7f899cfde9a499bb81c4498d10956c4f31ee3021f08a1b5","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","efea2dcd245af6f421d5d48a15159d0a97c6fcbf251bcc721e928eae3008a031","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","fc644a81a7601796e8c7e94724517f0056e9382ba0920b2aaa0529c449d6cba7"],"matched_occurrences":23,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240300545150976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6657,"pr_state":"open","pr_sha":"b7448d38fd31675bd8de4821dff7f0bde3aaee26","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T13:18:54Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1e2ad4cb17efd9ba1fea0f6f4ad2db737ff351bbc8b31b08a525010f3a32c17f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240300545150976","occurred_at":"2026-08-25T13:18:54Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092234666756018176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"b89541a3f2c5a6ac4c3ef08a4057390880510f4d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T12:56:26Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"44dc0b397c84b37814d1089b79f9bed7a9d540e2f6c5759328963549f8ae9af0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092234666756018176","occurred_at":"2026-08-25T12:56:26Z","failed_at":"e2e","signature_id":"fba360027133779a17d7b5f7a04eb5d7f56e70b4761fe51394db3cfa7dded15e","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048d20\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pmmkk7vxjzf8\" to finish creating: GET https://rp.j6018176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad1b70b6-b35f-4112-9ae9-1df6c2649ab2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad1b70b6-b35f-4112-9ae9-1df6c2649ab2\",\n \"name\": \"ad1b70b6-b35f-4112-9ae9-1df6c2649ab2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:00:12.864408043Z\",\n \"endTime\": \"2026-08-25T14:19:12.422414092Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048d20\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pmmkk7vxjzf8\" to finish creating: GET https://rp.j6018176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad1b70b6-b35f-4112-9ae9-1df6c2649ab2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad1b70b6-b35f-4112-9ae9-1df6c2649ab2\",\n \"name\": \"ad1b70b6-b35f-4112-9ae9-1df6c2649ab2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T14:00:12.864408043Z\",\n \"endTime\": \"2026-08-25T14:19:12.422414092Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4566,"pr_state":"open","pr_sha":"9749b2d351ded24fec2bb28c4ddc42fbadee5a36","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T12:43:00Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3d493ee6d13dedca00099bb37e134cdc48f068719551f3189cbd0301d733ca1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","failed_at":"e2e","signature_id":"88ef5278beac4e0b8897706c7e61f4cbb2e358d71c83d41a4ebf5e0e7c99a5ec","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009084e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qbnbmqvjsdcq\" to finish creating: GET https://rp.j8712960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0cf3357-f80d-46bd-801d-3a7369bbca29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0cf3357-f80d-46bd-801d-3a7369bbca29\",\n \"name\": \"f0cf3357-f80d-46bd-801d-3a7369bbca29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T13:45:24.403678541Z\",\n \"endTime\": \"2026-08-25T14:04:32.105380072Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009084e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qbnbmqvjsdcq\" to finish creating: GET https://rp.j8712960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0cf3357-f80d-46bd-801d-3a7369bbca29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0cf3357-f80d-46bd-801d-3a7369bbca29\",\n \"name\": \"f0cf3357-f80d-46bd-801d-3a7369bbca29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T13:45:24.403678541Z\",\n \"endTime\": \"2026-08-25T14:04:32.105380072Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"87b2a3a0342491f905b350c1f0ffd374a2ba8762c329d2e59266765446fb118b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","failed_at":"e2e","signature_id":"cba54857857016b020e86b6a86de63b4978237a179d12aeabfff1cca95a87fa0","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee5280\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kxmgj9sxlkxj\" to finish creating: GET https://rp.j8712960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/33987596-6be7-4dfe-aa71-8328c20f9b13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/33987596-6be7-4dfe-aa71-8328c20f9b13\",\n \"name\": \"33987596-6be7-4dfe-aa71-8328c20f9b13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T13:44:21.781571493Z\",\n \"endTime\": \"2026-08-25T14:03:22.119943491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee5280\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kxmgj9sxlkxj\" to finish creating: GET https://rp.j8712960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/33987596-6be7-4dfe-aa71-8328c20f9b13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/33987596-6be7-4dfe-aa71-8328c20f9b13\",\n \"name\": \"33987596-6be7-4dfe-aa71-8328c20f9b13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T13:44:21.781571493Z\",\n \"endTime\": \"2026-08-25T14:03:22.119943491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c70eccb939b4e4e7049456577be337ce974220cd4f78220fb4ee8646885c689a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","failed_at":"e2e","signature_id":"c3e9e8fbb2c80bbe0532378e51b9ba35394e8a18321158765e0e9d2c9531e792","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018f4220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-t52sl5qhgrmd\" to finish creating: GET https://rp.j8712960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b14f096f-1fdb-4e55-b7c3-07232f031770\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b14f096f-1fdb-4e55-b7c3-07232f031770\",\n \"name\": \"b14f096f-1fdb-4e55-b7c3-07232f031770\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T13:43:02.409731764Z\",\n \"endTime\": \"2026-08-25T14:02:02.069640925Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018f4220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-t52sl5qhgrmd\" to finish creating: GET https://rp.j8712960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b14f096f-1fdb-4e55-b7c3-07232f031770\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b14f096f-1fdb-4e55-b7c3-07232f031770\",\n \"name\": \"b14f096f-1fdb-4e55-b7c3-07232f031770\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T13:43:02.409731764Z\",\n \"endTime\": \"2026-08-25T14:02:02.069640925Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231217540763648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T12:42:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092224802956775424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6669,"pr_state":"open","pr_sha":"5c0ce63e7ccb9720b19764274e1ec873a56d9880","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T12:17:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092223656909344768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T12:12:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092220412426457088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6676,"pr_state":"open","pr_sha":"0b3e24fb93f95bc770e101def5b755481641225a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T11:59:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092218654312632320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6676,"pr_state":"open","pr_sha":"1460cb00a078b0dd153c1cf2916e7f3d95ef57bc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T11:52:53Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"21d80d9dcad2bc2bece162ca94633c16b4127535ca4362c283971c30f1c62f62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092218654312632320","occurred_at":"2026-08-25T11:52:53Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092215603107467264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T11:40:45Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"09c216e456b97beb2375a9d74d92692af0cc61c078b336c957c4c14c8d1aaecd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092215603107467264","occurred_at":"2026-08-25T11:40:45Z","failed_at":"e2e","signature_id":"2aaa34c36208117bcee63f75dc471d9e9468caff13d1659c2583035ece83c2fb","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001ac6140\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xsr5g7ds2s49\" to finish creating: GET https://rp.j7467264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c63b46d-df4e-4caf-b1a9-fee835ae340b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c63b46d-df4e-4caf-b1a9-fee835ae340b\",\n \"name\": \"3c63b46d-df4e-4caf-b1a9-fee835ae340b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T12:28:28.768318189Z\",\n \"endTime\": \"2026-08-25T12:47:36.715383913Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001ac6140\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xsr5g7ds2s49\" to finish creating: GET https://rp.j7467264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c63b46d-df4e-4caf-b1a9-fee835ae340b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c63b46d-df4e-4caf-b1a9-fee835ae340b\",\n \"name\": \"3c63b46d-df4e-4caf-b1a9-fee835ae340b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T12:28:28.768318189Z\",\n \"endTime\": \"2026-08-25T12:47:36.715383913Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092208871509594112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"5fdef9ddfd1ced2b8d1ba4a721a7a99e261c25b7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T11:14:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3a902502a2e9fd771fde7bbc097f60aadcfc8c7112936ad7f10d55a658316e77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092208871509594112","occurred_at":"2026-08-25T11:14:00Z","failed_at":"e2e","signature_id":"85a73bf3207678bf07196fd0dce16f63b867de9d5fd5b8dcc20aa62ad1433a40","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6cce0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pcmvmmpvpflg\" to finish creating: GET https://rp.j9594112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"name\": \"52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:59:13.55636864Z\",\n \"endTime\": \"2026-08-25T12:18:21.188692138Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6cce0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pcmvmmpvpflg\" to finish creating: GET https://rp.j9594112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"name\": \"52000f23-e3aa-465f-b2e6-2408f772e24f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:59:13.55636864Z\",\n \"endTime\": \"2026-08-25T12:18:21.188692138Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bd8944c9560147308a683885d199860a816c24f450fa26dcc2d571dd6c882239","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bd8944c9560147308a683885d199860a816c24f450fa26dcc2d571dd6c882239"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092208796284751872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6675,"pr_state":"closed","pr_sha":"0e91a6e0d063522f41d8522725a8bf02424f1086","final_merged_sha":"0e91a6e0d063522f41d8522725a8bf02424f1086","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T11:13:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092206223259602944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6674,"pr_state":"open","pr_sha":"69272e6a237fd9dc92b0fb2cf03c295a1401089f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T11:03:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6433,"pr_state":"open","pr_sha":"9ec65054f149d1b7507171a2d613008ef1bc9761","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:30:42Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"0f1319d3d089ca07a432fef2d18192ad7499440fbff468a4727b09865c4ae3db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","failed_at":"alert","signature_id":"f21b751e01203e4e27106dcb1932c067722032d928d9e8e29b55092b5008f5d9","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-25T11:37:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"345fcf25b932547d8e0ebfa5b9c113562e3ba7a8f931aa7486ce7540f0e2c4c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","failed_at":"alert","signature_id":"7ebd8243470fe8b709d9e6b75f707a2eff5fb9d0f87bf99b81e7e336da9429c9","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:52:32Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeNotReady\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"prometheus\", node=\"aks-system-30244119-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n Description: aks-system-30244119-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T11:52:32Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeNotReady\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"prometheus\", node=\"aks-system-30244119-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n Description: aks-system-30244119-vmss000000 has been unready for more than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired"}},{"row_id":"3d3b825e1928803252acc601754fe707f74af85110d9d3e69ab0ec264f7e355a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","failed_at":"alert","signature_id":"41f7f74865411c0e7be6f2be5d79bdefecee7e90cea861519501a2ae2711a449","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-rvqrk\", severity=\"warning\"\n Description: pod/ama-metrics-node-rvqrk in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-rvqrk\", severity=\"warning\"\n Description: pod/ama-metrics-node-rvqrk in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container collector has been in waiting state for longer than 1 hour.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-08-25T12:22:23Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6657,"pr_state":"open","pr_sha":"8b8c62cf35434a4402742ed59b61af5eb6ff1552","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:30:20Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"458c76cb0b4fc099f7ece4ff82898f99d90bee24ecab0ce91906559cdda0fc83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"665d04bdbe51dea0779e5498e680679376ee05757388a123b82798ebe3dc6751","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","failed_at":"alert","signature_id":"9f099708799bf3e2ac428bc43b19f24600de3b38fc32adfc9aa785b8a28ba4cc","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:39:31Z\n Ended: 2026-08-25T12:14:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5727872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdir7piesvgh51a20oka0qaplpe4b0t-x2t4h8s0j2c7z6a\", pod=\"router-c975cc75f-9rwjt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdir7piesvgh51a20oka0qaplpe4b0t-x2t4h8s0j2c7z6a/router-c975cc75f-9rwjt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:40:34Z\n Ended: 2026-08-25T11:58:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5727872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdirl616dc560s7ergbqgff41rhcqqc-x2z0i1i4i8f5x4u\", pod=\"router-5fbfd4cc9-n5cg7\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdirl616dc560s7ergbqgff41rhcqqc-x2z0i1i4i8f5x4u/router-5fbfd4cc9-n5cg7 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e19fdb1d437c34d5eb1edc3f85784d6bb98e2a06b0d86ca92a1083a00ed508c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","failed_at":"e2e","signature_id":"ba4575d0d05801721860d214c9ab20325f6c0ce684b7a86f371bbdb0b3c221f6","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009be400\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-m6f64pzjnj2b\" to finish creating: GET https://rp.j5727872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25febb2b-752c-4172-b76b-636cdbb8da75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25febb2b-752c-4172-b76b-636cdbb8da75\",\n \"name\": \"25febb2b-752c-4172-b76b-636cdbb8da75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:26:11.043207919Z\",\n \"endTime\": \"2026-08-25T11:45:13.698680289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009be400\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-m6f64pzjnj2b\" to finish creating: GET https://rp.j5727872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25febb2b-752c-4172-b76b-636cdbb8da75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/25febb2b-752c-4172-b76b-636cdbb8da75\",\n \"name\": \"25febb2b-752c-4172-b76b-636cdbb8da75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:26:11.043207919Z\",\n \"endTime\": \"2026-08-25T11:45:13.698680289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:29:20Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"18a1a42a6f98e8177f2ba93aac3935a8aca96569d1b4baa8c2c82cad7e0a5fd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"ed7279f8680b41f53bf9dea3800ff6d209dab4158b53dc3aa29303a69736db57","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-xxw9sj\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd07c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xxw9sj: failed waiting for cluster=\"cluster-zstream-4-21-xxw9sj\" in resourcegroup=\"rg-zstream-upgrade-4-21-c6fglssn9lfl\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2126240-ce19-4a5a-9d90-bf9bef38c389\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2126240-ce19-4a5a-9d90-bf9bef38c389\",\n \"name\": \"d2126240-ce19-4a5a-9d90-bf9bef38c389\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:06.200554482Z\",\n \"endTime\": \"2026-08-25T11:32:15.844191701Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-xxw9sj\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd07c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xxw9sj: failed waiting for cluster=\"cluster-zstream-4-21-xxw9sj\" in resourcegroup=\"rg-zstream-upgrade-4-21-c6fglssn9lfl\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2126240-ce19-4a5a-9d90-bf9bef38c389\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2126240-ce19-4a5a-9d90-bf9bef38c389\",\n \"name\": \"d2126240-ce19-4a5a-9d90-bf9bef38c389\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:06.200554482Z\",\n \"endTime\": \"2026-08-25T11:32:15.844191701Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"30b7797250e29fecdeaf5ecd417bad88cfe4ff3272b9c6e4d1da112714fd5b8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"cfec1836d28d37c357fa4cdaaa7a7e2cfa94f1863cd3c937f9210fab49a6b769","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c88360\u003e: \n failed to create HCP cluster ea-listgrtd5h: failed waiting for cluster=\"ea-listgrtd5h\" in resourcegroup=\"ea-list-xlnjgvtfpm84\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acc01fc7-72a8-4510-9e4a-d567598b9c1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acc01fc7-72a8-4510-9e4a-d567598b9c1d\",\n \"name\": \"acc01fc7-72a8-4510-9e4a-d567598b9c1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:12.096173667Z\",\n \"endTime\": \"2026-08-25T11:32:15.908926647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c88360\u003e: \n failed to create HCP cluster ea-listgrtd5h: failed waiting for cluster=\"ea-listgrtd5h\" in resourcegroup=\"ea-list-xlnjgvtfpm84\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acc01fc7-72a8-4510-9e4a-d567598b9c1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acc01fc7-72a8-4510-9e4a-d567598b9c1d\",\n \"name\": \"acc01fc7-72a8-4510-9e4a-d567598b9c1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:12.096173667Z\",\n \"endTime\": \"2026-08-25T11:32:15.908926647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"50b8cf0293b4da97b7ae545d9ad5221ab5ea587cb2b77a4170d710de6ed68b95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"941ca1754672c53670ae6f9235025ce03ee81e89e2ab50e5ebdc653a9104513b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef9640\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4hk8t569ldwn\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09ac2de1-d9ce-4cf8-bc9c-7c422d5d09b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09ac2de1-d9ce-4cf8-bc9c-7c422d5d09b6\",\n \"name\": \"09ac2de1-d9ce-4cf8-bc9c-7c422d5d09b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:43.911188079Z\",\n \"endTime\": \"2026-08-25T11:32:45.953478879Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef9640\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4hk8t569ldwn\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09ac2de1-d9ce-4cf8-bc9c-7c422d5d09b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09ac2de1-d9ce-4cf8-bc9c-7c422d5d09b6\",\n \"name\": \"09ac2de1-d9ce-4cf8-bc9c-7c422d5d09b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:43.911188079Z\",\n \"endTime\": \"2026-08-25T11:32:45.953478879Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b1aba0bc3c8ec498c2a499d184cf7f242ee00dce295c317b684c019b3566cbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"b43fff9cd9bd036194d64006e2bedb42a5de47a08b4bb6a5327c7a329e3576da","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6eca0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qtr6p2j9mh46\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d62ea837-b979-4ca7-86a1-f20df85fd7b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d62ea837-b979-4ca7-86a1-f20df85fd7b1\",\n \"name\": \"d62ea837-b979-4ca7-86a1-f20df85fd7b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:16.876555859Z\",\n \"endTime\": \"2026-08-25T11:32:25.915902762Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6eca0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qtr6p2j9mh46\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d62ea837-b979-4ca7-86a1-f20df85fd7b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d62ea837-b979-4ca7-86a1-f20df85fd7b1\",\n \"name\": \"d62ea837-b979-4ca7-86a1-f20df85fd7b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:16.876555859Z\",\n \"endTime\": \"2026-08-25T11:32:25.915902762Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"74b4a0692bc18606ddad4e54db53e93ea156280ea8b86a042e82d8ddca00df33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"f2cb49f19339d4683d5c2bd5fd290c1a0f9cba1b7b1446ce110a22ee9a0d5a35","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-t7drrl95cd8z/cluster-candidate-4-20-t2lbd9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a36920\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t2lbd9: failed waiting for cluster=\"cluster-candidate-4-20-t2lbd9\" in resourcegroup=\"rg-candidate-4-20-t7drrl95cd8z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"name\": \"579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:01.56047746Z\",\n \"endTime\": \"2026-08-25T11:32:05.952606176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-t7drrl95cd8z/cluster-candidate-4-20-t2lbd9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a36920\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t2lbd9: failed waiting for cluster=\"cluster-candidate-4-20-t2lbd9\" in resourcegroup=\"rg-candidate-4-20-t7drrl95cd8z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"name\": \"579deeb9-f336-4f04-8b2e-b7d680266538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:01.56047746Z\",\n \"endTime\": \"2026-08-25T11:32:05.952606176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2fc8daa06dcd0810c6da5b7d738c6be1c97e759c090dbe32d9900028e598f859","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8838108d357f1108afd8e2c46a96f4631942083393c73ba93d232ecf23040614","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"da7c4bc1633eca36853e5843a03bffeb0f874d7d6c7c76694adad89844da3845","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000978da0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zn5xf6rhmm5z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4881145-71b4-45d4-ac59-0367dec0a789\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4881145-71b4-45d4-ac59-0367dec0a789\",\n \"name\": \"e4881145-71b4-45d4-ac59-0367dec0a789\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:11.121536159Z\",\n \"endTime\": \"2026-08-25T11:32:15.953394704Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000978da0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zn5xf6rhmm5z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4881145-71b4-45d4-ac59-0367dec0a789\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4881145-71b4-45d4-ac59-0367dec0a789\",\n \"name\": \"e4881145-71b4-45d4-ac59-0367dec0a789\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:11.121536159Z\",\n \"endTime\": \"2026-08-25T11:32:15.953394704Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"99aa3a0041b86a67e3da57f2a96cafed393ec8060f12f9b939e9c4fa04c5c779","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"99dd3763374e539920c960007cd1b14c1c67fd84b0f8ee47d35b2abd7ec3c4a2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-k54h9d\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9cf00\u003e: \n failed to create HCP cluster cluster-zstream-4-22-k54h9d: failed waiting for cluster=\"cluster-zstream-4-22-k54h9d\" in resourcegroup=\"rg-zstream-upgrade-4-22-r5wjf2g99fnb\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fe8d963-e967-44a6-bfa1-19ebb4ebffbc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fe8d963-e967-44a6-bfa1-19ebb4ebffbc\",\n \"name\": \"9fe8d963-e967-44a6-bfa1-19ebb4ebffbc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:12:56.985755901Z\",\n \"endTime\": \"2026-08-25T11:32:05.9413244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-k54h9d\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9cf00\u003e: \n failed to create HCP cluster cluster-zstream-4-22-k54h9d: failed waiting for cluster=\"cluster-zstream-4-22-k54h9d\" in resourcegroup=\"rg-zstream-upgrade-4-22-r5wjf2g99fnb\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fe8d963-e967-44a6-bfa1-19ebb4ebffbc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fe8d963-e967-44a6-bfa1-19ebb4ebffbc\",\n \"name\": \"9fe8d963-e967-44a6-bfa1-19ebb4ebffbc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:12:56.985755901Z\",\n \"endTime\": \"2026-08-25T11:32:05.9413244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b65c045cb8849ed4e8602f7776722b88a39b207d87c39ac475eacfd5462b875a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"d833ff037f5488f1fab9b945d4f55d14924d6973d61766c5187da39c788703a1","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-6nw9hm with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000892700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-6nw9hm: failed waiting for cluster=\"np-version-upgrade-cluster-6nw9hm\" in resourcegroup=\"rg-np-version-upgrade-6nw9hm-sbc4vzg69snt\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"name\": \"dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:18.056611915Z\",\n \"endTime\": \"2026-08-25T11:32:25.82846965Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-6nw9hm with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000892700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-6nw9hm: failed waiting for cluster=\"np-version-upgrade-cluster-6nw9hm\" in resourcegroup=\"rg-np-version-upgrade-6nw9hm-sbc4vzg69snt\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"name\": \"dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:18.056611915Z\",\n \"endTime\": \"2026-08-25T11:32:25.82846965Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d3cb18cef27e3cf90db54bfa014f598a9a6224d89a4e03a3b756b5280b317733","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"e2e","signature_id":"0d1f7426cf173fcfa700b32a523143bec8203aa837293f2ac55c31004cab129a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009101c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-vzm85c2qbsgd\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01cf43dd-3011-4a8d-9caf-ea328f7d5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01cf43dd-3011-4a8d-9caf-ea328f7d5fc4\",\n \"name\": \"01cf43dd-3011-4a8d-9caf-ea328f7d5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:23.203350811Z\",\n \"endTime\": \"2026-08-25T11:32:25.896029624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009101c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-vzm85c2qbsgd\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01cf43dd-3011-4a8d-9caf-ea328f7d5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01cf43dd-3011-4a8d-9caf-ea328f7d5fc4\",\n \"name\": \"01cf43dd-3011-4a8d-9caf-ea328f7d5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:13:23.203350811Z\",\n \"endTime\": \"2026-08-25T11:32:25.896029624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ef8364fbbd0441ef851ad8fa88506610e1aa20da84a5831e4271ea60a6757ee6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","failed_at":"alert","signature_id":"9323f34e5dc7624ff70491391f71624b8fccbc2b4b0fdf3ebe01e36965be3373","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 70 time(s) (20 unknown, 50 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:22:16Z\n Ended: 2026-08-25T11:28:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilgqdj36sal4q4e9a57hvh6ajgfbp-j3p9v3z4j5r6p2w\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilgqdj36sal4q4e9a57hvh6ajgfbp-j3p9v3z4j5r6p2w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdili48rqlgtv5ms3vjppa997f6gctd-ea-listgrtd5h\", pod=\"cluster-api-57f4cccc64-zkpfw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdili48rqlgtv5ms3vjppa997f6gctd-ea-listgrtd5h/cluster-api-57f4cccc64-zkpfw has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilggqeajkcm0ivagj6qovcves1q5b-a1f9o5m3y1a2v1u\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilggqeajkcm0ivagj6qovcves1q5b-a1f9o5m3y1a2v1u/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilqfi7gmv0s4r7i9nnu81qe3sfhmu-o7a7g0l5g7d0e6d\", pod=\"control-plane-operator-5846985856-dwpdq\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilqfi7gmv0s4r7i9nnu81qe3sfhmu-o7a7g0l5g7d0e6d/control-plane-operator-5846985856-dwpdq has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:31:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilgko7uh068qmibb2p2nl80hmcr2t-sre-hcp-cluster\", pod=\"kube-apiserver-6fff59d9b9-r7c94\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilgko7uh068qmibb2p2nl80hmcr2t-sre-hcp-cluster/kube-apiserver-6fff59d9b9-r7c94 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiljh6ae8gvdi928aqg7igno5v6m4a-m9l2r5o1n9x5m1r\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiljh6ae8gvdi928aqg7igno5v6m4a-m9l2r5o1n9x5m1r/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilimj0peupdgmcd9ptu4jvrh6qg2d-c3m2x0d8u8p4q0n\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilimj0peupdgmcd9ptu4jvrh6qg2d-c3m2x0d8u8p4q0n/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdile8siqsvbkl4sfirmman2o5ffnof-k8a3l8q9y2b8h3z\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdile8siqsvbkl4sfirmman2o5ffnof-k8a3l8q9y2b8h3z/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:30:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdili0tb2m5lpededoul600v6dh93b7-j9q2y1u3s0w4r7x\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdili0tb2m5lpededoul600v6dh93b7-j9q2y1u3s0w4r7x/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiltvi8g0n5p1koouov5n23ea3c448-g6d6q7a2u2x0l7v\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiltvi8g0n5p1koouov5n23ea3c448-g6d6q7a2u2x0l7v/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiltvi8g0n5p1koouov5n23ea3c448-g6d6q7a2u2x0l7v\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiltvi8g0n5p1koouov5n23ea3c448-g6d6q7a2u2x0l7v/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T11:23:15Z\n Ended: 2026-08-25T11:30:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilm4kp8ht0oeq4rrf02uregq5naqb-r1e8l8q2p0u8h1y\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilm4kp8ht0oeq4rrf02uregq5naqb-r1e8l8q2p0u8h1y/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T11:23:16Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilqm74gj28677ca8klu0le33fadfb-c6e6h8e6t5t1g7r\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilqm74gj28677ca8klu0le33fadfb-c6e6h8e6t5t1g7r/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T11:23:16Z\n Ended: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilf9lrljchocg563vtvvjb0a24i8h-i5t4p8s6m7u7k8a\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilf9lrljchocg563vtvvjb0a24i8h-i5t4p8s6m7u7k8a/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T11:24:14Z\n Ended: 2026-08-25T11:30:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilkvrcjt48hulqqgsqhblsbqojees-p0x0o6d8w2v9t8p\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilkvrcjt48hulqqgsqhblsbqojees-p0x0o6d8w2v9t8p/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T11:24:14Z\n Ended: 2026-08-25T11:30:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdili0ibn99qtvs4jjc0tppu5v61opj-d3v7t2v2y2v6h1k\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdili0ibn99qtvs4jjc0tppu5v61opj-d3v7t2v2y2v6h1k/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T11:24:14Z\n Ended: 2026-08-25T11:30:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilj8q9l3vpagobot4o90tggf7475q-b0e5d6r8d2f9p5i\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilj8q9l3vpagobot4o90tggf7475q-b0e5d6r8d2f9p5i/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T11:24:14Z\n Ended: 2026-08-25T11:30:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilgq8asvq19tc9m6el9uvbgv8ngs6-g7b1y4g7p8f2l1r\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilgq8asvq19tc9m6el9uvbgv8ngs6-g7b1y4g7p8f2l1r/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T11:25:15Z\n Ended: 2026-08-25T11:31:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdilegalafbqc4itqpt0up25tbij7h7-i7l0h0h4g1d6r4d\", pod=\"kube-apiserver-75fcd8657-xclt4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdilegalafbqc4itqpt0up25tbij7h7-i7l0h0h4g1d6r4d/kube-apiserver-75fcd8657-xclt4 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-25T11:25:15Z\n Ended: 2026-08-25T11:31:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0982912-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdile6vap1e17rrbb0clqfaohcoisog-u2k1x5w3i5n2n6p\", pod=\"kube-apiserver-6f5cf68c44-rbm77\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdile6vap1e17rrbb0clqfaohcoisog-u2k1x5w3i5n2n6p/kube-apiserver-6f5cf68c44-rbm77 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","2fc8daa06dcd0810c6da5b7d738c6be1c97e759c090dbe32d9900028e598f859","30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"b89541a3f2c5a6ac4c3ef08a4057390880510f4d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:22:57Z"},"failed_at":["alert","e2e"],"failed_test_count":29,"occurrences":[{"row_id":"06bd885f98f4d75fe5164448856f263034b4e65d301eb1e2e2c529b430a18a92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"c23ab9a03bfcf50206e6f81c6de07ad7f3b1320c629158dce17b675a8a9b9d83","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zhhnpcphpl7r/cluster-candidate-4-21-tcc8ch should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2aa60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tcc8ch: failed waiting for cluster=\"cluster-candidate-4-21-tcc8ch\" in resourcegroup=\"rg-candidate-4-21-zhhnpcphpl7r\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\",\n \"name\": \"06c23c7d-cba1-433b-881e-637949594df3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.520227726Z\",\n \"endTime\": \"2026-08-25T11:20:37.842143923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zhhnpcphpl7r/cluster-candidate-4-21-tcc8ch should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2aa60\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tcc8ch: failed waiting for cluster=\"cluster-candidate-4-21-tcc8ch\" in resourcegroup=\"rg-candidate-4-21-zhhnpcphpl7r\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\",\n \"name\": \"06c23c7d-cba1-433b-881e-637949594df3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.520227726Z\",\n \"endTime\": \"2026-08-25T11:20:37.842143923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2fc8daa06dcd0810c6da5b7d738c6be1c97e759c090dbe32d9900028e598f859","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"07c16e8ad4723b5a513aafd34b62aa5294b5a42b77629983bd2b0f6de5078ce2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"alert","signature_id":"af8dcfdf2cbedd9372f04cbd90abd56a127bdc4079bafb1ef1913edb6177347d","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T12:06:25Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j9005312-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9005312/providers/microsoft.monitor/accounts/services-j9005312\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"09e24f5ba6973c1448dc4351833e6e1ab10339065cfd9c392555cbdb0f9c97dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"e814323bff83120ddf30b4df3d591da6897add0520ab77b9aece5773322a790f","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001076940\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-zlx9rbktl8hf\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12f586e0-5a79-48d8-98cc-d9683f3d6329\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12f586e0-5a79-48d8-98cc-d9683f3d6329\",\n \"name\": \"12f586e0-5a79-48d8-98cc-d9683f3d6329\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:39.436649403Z\",\n \"endTime\": \"2026-08-25T11:20:48.141412603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, community-operators-catalog, azure-cloud-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, cluster-version-operator, cluster-storage-operator, redhat-marketplace-catalog, catalog-operator, olm-operator, hosted-cluster-config-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001076940\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-zlx9rbktl8hf\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12f586e0-5a79-48d8-98cc-d9683f3d6329\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12f586e0-5a79-48d8-98cc-d9683f3d6329\",\n \"name\": \"12f586e0-5a79-48d8-98cc-d9683f3d6329\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:39.436649403Z\",\n \"endTime\": \"2026-08-25T11:20:48.141412603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, community-operators-catalog, azure-cloud-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-operators-catalog, cluster-version-operator, cluster-storage-operator, redhat-marketplace-catalog, catalog-operator, olm-operator, hosted-cluster-config-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"17850bb647b53f8359663e49cc63aa9f5fceacc99bc0e17d55189894e66524cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"83e815c22c5353bc66d4aad1ca57dc77c126e34a396f19d17d4961aa8e291e1b","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00069e2c0\u003e: \n failed to create HCP cluster np-dg-minor-zd5jtq: failed waiting for cluster=\"np-dg-minor-zd5jtq\" in resourcegroup=\"rg-np-dg-minor-zd5jtq-kkl7vl8r2p4z\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"name\": \"20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.680263097Z\",\n \"endTime\": \"2026-08-25T11:20:37.924037446Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00069e2c0\u003e: \n failed to create HCP cluster np-dg-minor-zd5jtq: failed waiting for cluster=\"np-dg-minor-zd5jtq\" in resourcegroup=\"rg-np-dg-minor-zd5jtq-kkl7vl8r2p4z\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"name\": \"20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.680263097Z\",\n \"endTime\": \"2026-08-25T11:20:37.924037446Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c20cd2f0949614f5b5bf1268024605c4b2d785d2673f7df63625d3b9d6e2a09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"30b876ac66882fc0e335572b0826b8ec58b54d7e36dc27655838b99b58cd8eb0","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000889040\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-794c9cl949m2\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8741a29d-3a29-4b25-9cca-bfb6b98db145\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8741a29d-3a29-4b25-9cca-bfb6b98db145\",\n \"name\": \"8741a29d-3a29-4b25-9cca-bfb6b98db145\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:20.170201007Z\",\n \"endTime\": \"2026-08-25T11:20:27.872895981Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000889040\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-794c9cl949m2\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8741a29d-3a29-4b25-9cca-bfb6b98db145\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8741a29d-3a29-4b25-9cca-bfb6b98db145\",\n \"name\": \"8741a29d-3a29-4b25-9cca-bfb6b98db145\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:20.170201007Z\",\n \"endTime\": \"2026-08-25T11:20:27.872895981Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f3e7f9359ecac53969a85daf734cbbce4163339e717a48493b0d7e76c43c2bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"0f33bde12ddadf9e7c9bfc135e1acbf84861720d826fda54f0ad89d0271fe465","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:122]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000943300\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-4hldwq4wn65q\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b775fba-4326-448d-9ed3-d84953f5f41e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b775fba-4326-448d-9ed3-d84953f5f41e\",\n \"name\": \"7b775fba-4326-448d-9ed3-d84953f5f41e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:18:57.616565456Z\",\n \"endTime\": \"2026-08-25T11:38:04.521702463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:122]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000943300\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-4hldwq4wn65q\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b775fba-4326-448d-9ed3-d84953f5f41e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b775fba-4326-448d-9ed3-d84953f5f41e\",\n \"name\": \"7b775fba-4326-448d-9ed3-d84953f5f41e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:18:57.616565456Z\",\n \"endTime\": \"2026-08-25T11:38:04.521702463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"49dc5e6ebc75da63773c3239e8e4ae393e6d6485842f01903505754d1debccfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"82e0621d077685d0022747285dcb794c36f142bd42f8f8a172a77dcd55aad3e9","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010427e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-scdxnxk455c9\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb03150b-7aa4-4977-bdd5-3bf252c14ed8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb03150b-7aa4-4977-bdd5-3bf252c14ed8\",\n \"name\": \"eb03150b-7aa4-4977-bdd5-3bf252c14ed8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.520597523Z\",\n \"endTime\": \"2026-08-25T11:20:37.903541851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010427e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-scdxnxk455c9\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb03150b-7aa4-4977-bdd5-3bf252c14ed8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb03150b-7aa4-4977-bdd5-3bf252c14ed8\",\n \"name\": \"eb03150b-7aa4-4977-bdd5-3bf252c14ed8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.520597523Z\",\n \"endTime\": \"2026-08-25T11:20:37.903541851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b5269be28090a916c8fd3a0a3a523fb73cb5f3b36c124fb31c9e6f604e87beb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"f83b12586d201faafc5d77e61170761a01913b4aa6e23e7374f34e7086cab7fc","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011325d0\u003e: \n GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"name\": \"642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:23.530391484Z\",\n \"endTime\": \"2026-08-25T11:22:27.900506106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011325d0\u003e: \n GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"name\": \"642ab867-7117-49d1-9205-fcc81b8fcac8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:23.530391484Z\",\n \"endTime\": \"2026-08-25T11:22:27.900506106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0371ff83ff8ae672d5d38be2f67ae6f726e275d3db1d866178e427376320fd8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"53b9139ef87b53e0d787fbc532c614b23e7948b52733465235023a268985fb36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"d6949fb19ef2eea4676f8abc14bdf9d0083de17b97edd1d98d989d65f25a63ef","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4eac0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-dvdtjf96rsnl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf5319d7-d2c8-4242-876b-f8a97dfb3996\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf5319d7-d2c8-4242-876b-f8a97dfb3996\",\n \"name\": \"bf5319d7-d2c8-4242-876b-f8a97dfb3996\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:39.266118595Z\",\n \"endTime\": \"2026-08-25T11:20:48.041047422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4eac0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-dvdtjf96rsnl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf5319d7-d2c8-4242-876b-f8a97dfb3996\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf5319d7-d2c8-4242-876b-f8a97dfb3996\",\n \"name\": \"bf5319d7-d2c8-4242-876b-f8a97dfb3996\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:39.266118595Z\",\n \"endTime\": \"2026-08-25T11:20:48.041047422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"596077f87500d36224732a93f09988a0355c82dd09a4283fc5d6ed8aded6896d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"d36de0542872df4f6d63fdbe3389505290d5b0baebed82cdd90fedf806ec3e01","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-j5vbtp\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00119a640\u003e: \n failed to create HCP cluster cluster-zstream-4-20-j5vbtp: failed waiting for cluster=\"cluster-zstream-4-20-j5vbtp\" in resourcegroup=\"rg-zstream-upgrade-4-20-xln7ltx8rz8n\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/141fab79-05e7-48e2-a785-31f4588cc106\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/141fab79-05e7-48e2-a785-31f4588cc106\",\n \"name\": \"141fab79-05e7-48e2-a785-31f4588cc106\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.143019098Z\",\n \"endTime\": \"2026-08-25T11:20:37.940391987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-j5vbtp\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00119a640\u003e: \n failed to create HCP cluster cluster-zstream-4-20-j5vbtp: failed waiting for cluster=\"cluster-zstream-4-20-j5vbtp\" in resourcegroup=\"rg-zstream-upgrade-4-20-xln7ltx8rz8n\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/141fab79-05e7-48e2-a785-31f4588cc106\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/141fab79-05e7-48e2-a785-31f4588cc106\",\n \"name\": \"141fab79-05e7-48e2-a785-31f4588cc106\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:30.143019098Z\",\n \"endTime\": \"2026-08-25T11:20:37.940391987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d59bf7371ae9ac338d31df7cc4300296498813da701023d1e18eeaefa703a1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"1e5d4e64aee3bb0b78e1fb3308473036771fbfa7bf224bebfc3e1ddb47aca9c8","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc9280\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ldt9sxjdns5j\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/207b6e6b-d81f-4ae8-a52d-235354977a78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/207b6e6b-d81f-4ae8-a52d-235354977a78\",\n \"name\": \"207b6e6b-d81f-4ae8-a52d-235354977a78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:20:21.066955417Z\",\n \"endTime\": \"2026-08-25T11:39:24.495248134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc9280\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ldt9sxjdns5j\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/207b6e6b-d81f-4ae8-a52d-235354977a78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/207b6e6b-d81f-4ae8-a52d-235354977a78\",\n \"name\": \"207b6e6b-d81f-4ae8-a52d-235354977a78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:20:21.066955417Z\",\n \"endTime\": \"2026-08-25T11:39:24.495248134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5f4649e5d406723f723ff9cf77fdded4e514567e4d61a3ba5b686ded152137f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"1b1a509582594e9612da11037861a2814fe682ec6e6e3e82e70696765264d057","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a84540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qsmsgbz4r9bl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"name\": \"eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:10.235142906Z\",\n \"endTime\": \"2026-08-25T11:22:17.989968047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a84540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qsmsgbz4r9bl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"name\": \"eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:10.235142906Z\",\n \"endTime\": \"2026-08-25T11:22:17.989968047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6acd57b7ba376f7c59b823c0a136783cfc6f6eb4667864cc92c6202cb24ce7a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"31a5fbbcc1ff5e782a4df57bfa09d31a765c01a8df5db6afadfacac658e620d6","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8c1c0\u003e: \n failed to create HCP cluster ea-listwl9qrq: failed waiting for cluster=\"ea-listwl9qrq\" in resourcegroup=\"ea-list-cs4nbh2vqlnx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"name\": \"ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.900770335Z\",\n \"endTime\": \"2026-08-25T11:20:27.863710783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8c1c0\u003e: \n failed to create HCP cluster ea-listwl9qrq: failed waiting for cluster=\"ea-listwl9qrq\" in resourcegroup=\"ea-list-cs4nbh2vqlnx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"name\": \"ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.900770335Z\",\n \"endTime\": \"2026-08-25T11:20:27.863710783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70463e98273065a1f3d888689251bd9b7c6e3052dd6ca43da1508c744a639e1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"7ba4d18cd6273c16cfe80bffe517d25785f0d11a2228c4383f25ed32d2c438d6","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000338280\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jjh86gpxzmk5\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4412341-97ae-4d89-b70d-d190163e6b84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4412341-97ae-4d89-b70d-d190163e6b84\",\n \"name\": \"f4412341-97ae-4d89-b70d-d190163e6b84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:14:26.528402468Z\",\n \"endTime\": \"2026-08-25T11:33:34.597854788Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000338280\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jjh86gpxzmk5\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4412341-97ae-4d89-b70d-d190163e6b84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4412341-97ae-4d89-b70d-d190163e6b84\",\n \"name\": \"f4412341-97ae-4d89-b70d-d190163e6b84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:14:26.528402468Z\",\n \"endTime\": \"2026-08-25T11:33:34.597854788Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e6d668bcb390d304aaea782b41572442a21fd826499a10d51d28cc6194b9c48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"befa32cb5bdc16812833dab6b4ef32700219a3216e83ebcad65e2edc52d10b6c","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-nmv2vp with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00115a280\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-nmv2vp: failed waiting for cluster=\"np-version-upgrade-cluster-nmv2vp\" in resourcegroup=\"rg-np-version-upgrade-nmv2vp-8jkn6dmxnnxc\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/309d309a-4846-475e-aa8f-a51a3a3cc734\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/309d309a-4846-475e-aa8f-a51a3a3cc734\",\n \"name\": \"309d309a-4846-475e-aa8f-a51a3a3cc734\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:41.544397084Z\",\n \"endTime\": \"2026-08-25T11:20:48.26115411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-nmv2vp with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00115a280\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-nmv2vp: failed waiting for cluster=\"np-version-upgrade-cluster-nmv2vp\" in resourcegroup=\"rg-np-version-upgrade-nmv2vp-8jkn6dmxnnxc\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/309d309a-4846-475e-aa8f-a51a3a3cc734\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/309d309a-4846-475e-aa8f-a51a3a3cc734\",\n \"name\": \"309d309a-4846-475e-aa8f-a51a3a3cc734\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:41.544397084Z\",\n \"endTime\": \"2026-08-25T11:20:48.26115411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e42ec7688c93a3c154a45ac10b0d9c9804a36417c87f09f7525aa7e27f6f80d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"45742a02b7e0c32d86b97863f4ea299dc6196292e7ab7de047ae62dec5023312","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a20320\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jxtcfnn8kwsp\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/42062015-dca9-4bc0-aad5-ab4a1b9d09de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/42062015-dca9-4bc0-aad5-ab4a1b9d09de\",\n \"name\": \"42062015-dca9-4bc0-aad5-ab4a1b9d09de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.494264153Z\",\n \"endTime\": \"2026-08-25T11:20:37.948177471Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-storage-operator, cluster-network-operator, olm-operator, certified-operators-catalog, dns-operator, catalog-operator, redhat-marketplace-catalog, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a20320\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jxtcfnn8kwsp\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/42062015-dca9-4bc0-aad5-ab4a1b9d09de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/42062015-dca9-4bc0-aad5-ab4a1b9d09de\",\n \"name\": \"42062015-dca9-4bc0-aad5-ab4a1b9d09de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.494264153Z\",\n \"endTime\": \"2026-08-25T11:20:37.948177471Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-storage-operator, cluster-network-operator, olm-operator, certified-operators-catalog, dns-operator, catalog-operator, redhat-marketplace-catalog, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"90789bcb5bdea78f2add2649cea2eafaf58d50018bb8c7bc1ee422eec5a30835","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"722d515e29fb9ec826abf197c41290ecc658816cb59a8a63c0cebddb0232cee7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae8400\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-264hwln96qbw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2ee6a72-aa37-4fa5-8b5f-6440eb7438e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2ee6a72-aa37-4fa5-8b5f-6440eb7438e4\",\n \"name\": \"f2ee6a72-aa37-4fa5-8b5f-6440eb7438e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.596159853Z\",\n \"endTime\": \"2026-08-25T11:20:48.273802602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, catalog-operator, community-operators-catalog, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae8400\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-264hwln96qbw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2ee6a72-aa37-4fa5-8b5f-6440eb7438e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2ee6a72-aa37-4fa5-8b5f-6440eb7438e4\",\n \"name\": \"f2ee6a72-aa37-4fa5-8b5f-6440eb7438e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.596159853Z\",\n \"endTime\": \"2026-08-25T11:20:48.273802602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, redhat-operators-catalog, certified-operators-catalog, catalog-operator, community-operators-catalog, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5dac47e5028b2fa262bcb00f704ee459c92667cea311d819d0565709c184cf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"62b66f3cda1a1b03c5a9f22a0c0fb4bfb068df7258a21d08a9456f0d47c984ee","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--2zj5qrnqfwdn/cluster-hs-pre-4n4p2k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e89c0\u003e: \n failed to create HCP cluster cluster-hs-pre-4n4p2k: failed waiting for cluster=\"cluster-hs-pre-4n4p2k\" in resourcegroup=\"rg-hs-pre--2zj5qrnqfwdn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"name\": \"7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:28.656859519Z\",\n \"endTime\": \"2026-08-25T11:20:37.93508057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, cluster-image-registry-operator, packageserver, certified-operators-catalog, hosted-cluster-config-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, catalog-operator, cluster-network-operator, machine-approver, csi-snapshot-controller-operator, community-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--2zj5qrnqfwdn/cluster-hs-pre-4n4p2k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e89c0\u003e: \n failed to create HCP cluster cluster-hs-pre-4n4p2k: failed waiting for cluster=\"cluster-hs-pre-4n4p2k\" in resourcegroup=\"rg-hs-pre--2zj5qrnqfwdn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"name\": \"7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:28.656859519Z\",\n \"endTime\": \"2026-08-25T11:20:37.93508057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, cluster-image-registry-operator, packageserver, certified-operators-catalog, hosted-cluster-config-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, catalog-operator, cluster-network-operator, machine-approver, csi-snapshot-controller-operator, community-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c97c7e10e9984956816f2978cee06c0347355fb43e2e18dbc518e10a50fd697c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a929507c5ee39b1908b78025236aa8449bdc3a95929d092b2c826895b8b9579f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"bd627bb7dd0c1e61533ffcaff0de1677c51817a542e05bd6bac955235f59715c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000740280\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-66vggwmcrfkm\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ed9a849-4e1a-47be-9fbf-9892b4e55909\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ed9a849-4e1a-47be-9fbf-9892b4e55909\",\n \"name\": \"5ed9a849-4e1a-47be-9fbf-9892b4e55909\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:00:58.915429667Z\",\n \"endTime\": \"2026-08-25T11:20:07.870969411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000740280\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-66vggwmcrfkm\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ed9a849-4e1a-47be-9fbf-9892b4e55909\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ed9a849-4e1a-47be-9fbf-9892b4e55909\",\n \"name\": \"5ed9a849-4e1a-47be-9fbf-9892b4e55909\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:00:58.915429667Z\",\n \"endTime\": \"2026-08-25T11:20:07.870969411Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9bca266dd23aff42fc29e3e0b362e6b238485922ff7c674fa9443f836dd6576","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"a12a937f6ae89e3736e75bd6e94756e9cbd733f875fec3c20ccc3e4632e8284a","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fac240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k5wdpllgjssx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a6c32ee-264c-4461-998b-e5b7a3d5b8ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a6c32ee-264c-4461-998b-e5b7a3d5b8ad\",\n \"name\": \"8a6c32ee-264c-4461-998b-e5b7a3d5b8ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.221887867Z\",\n \"endTime\": \"2026-08-25T11:20:48.105155011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fac240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k5wdpllgjssx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a6c32ee-264c-4461-998b-e5b7a3d5b8ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a6c32ee-264c-4461-998b-e5b7a3d5b8ad\",\n \"name\": \"8a6c32ee-264c-4461-998b-e5b7a3d5b8ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.221887867Z\",\n \"endTime\": \"2026-08-25T11:20:48.105155011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ab6a40dc072372d7cc4df13c089920a599c01cd511a64a4c28352feaa0adc95d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"0fa98435175007c5311cd1a7a6087ef4be15b52a462c15b286367a764bab1677","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c409e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g7htcsxqgjh8\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"name\": \"dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:51.512570533Z\",\n \"endTime\": \"2026-08-25T11:23:58.061151727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c409e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g7htcsxqgjh8\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"name\": \"dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:51.512570533Z\",\n \"endTime\": \"2026-08-25T11:23:58.061151727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b791747c12b6584495fbfd391fc2865f8a3523eac4e49ef2edeb45ac55827ee6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"af3ee147c3245ccf45bdc2deacd994348ca6f9c719f39a02ed738a5c4ec6e6d7","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010626e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-6xkgnbgv84t9\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7966876-e222-4b01-94a7-f89b2c83e042\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7966876-e222-4b01-94a7-f89b2c83e042\",\n \"name\": \"f7966876-e222-4b01-94a7-f89b2c83e042\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:02.696486003Z\",\n \"endTime\": \"2026-08-25T11:23:08.046324509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, csi-snapshot-controller-operator, packageserver, catalog-operator, cluster-version-operator, olm-operator, cluster-network-operator, hosted-cluster-config-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010626e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-6xkgnbgv84t9\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7966876-e222-4b01-94a7-f89b2c83e042\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7966876-e222-4b01-94a7-f89b2c83e042\",\n \"name\": \"f7966876-e222-4b01-94a7-f89b2c83e042\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:02.696486003Z\",\n \"endTime\": \"2026-08-25T11:23:08.046324509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, csi-snapshot-controller-operator, packageserver, catalog-operator, cluster-version-operator, olm-operator, cluster-network-operator, hosted-cluster-config-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"be96b2f8662d4863b195647c5984294b893a0019bba5651910f39e4ee8db3e71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"90684796a4aec3250755dd0c0757e80b9dd9a4d843996fedaf91a968864f7cc1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-f9drnxf8872v/cluster-candidate-4-20-7gm7rd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b08b00\u003e: \n failed to create HCP cluster cluster-candidate-4-20-7gm7rd: failed waiting for cluster=\"cluster-candidate-4-20-7gm7rd\" in resourcegroup=\"rg-candidate-4-20-f9drnxf8872v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0d278c3-b1dc-43c0-9907-4e3f268841ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0d278c3-b1dc-43c0-9907-4e3f268841ce\",\n \"name\": \"c0d278c3-b1dc-43c0-9907-4e3f268841ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.794018971Z\",\n \"endTime\": \"2026-08-25T11:20:37.949376228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, ingress-operator, openshift-controller-manager, dns-operator, csi-snapshot-controller-operator, olm-operator, cluster-version-operator, packageserver, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-network-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-f9drnxf8872v/cluster-candidate-4-20-7gm7rd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b08b00\u003e: \n failed to create HCP cluster cluster-candidate-4-20-7gm7rd: failed waiting for cluster=\"cluster-candidate-4-20-7gm7rd\" in resourcegroup=\"rg-candidate-4-20-f9drnxf8872v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0d278c3-b1dc-43c0-9907-4e3f268841ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0d278c3-b1dc-43c0-9907-4e3f268841ce\",\n \"name\": \"c0d278c3-b1dc-43c0-9907-4e3f268841ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:29.794018971Z\",\n \"endTime\": \"2026-08-25T11:20:37.949376228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, ingress-operator, openshift-controller-manager, dns-operator, csi-snapshot-controller-operator, olm-operator, cluster-version-operator, packageserver, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-network-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c3a147075cab3c2f5641c92de44f1f7acde7705c88522439f52c898cf77f7d5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"alert","signature_id":"1399061bb15ba2afe3a3d49ef0c1c03b496dabeb92275fbed52214cd772f172d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 69 time(s) (22 unknown, 47 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:12:48Z\n Ended: 2026-08-25T11:18:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2e8otp76oitb1jh1f4qm5u2spvk-w1t1q3s6x1e1x5d\", pod=\"router-577849c6f-7p6c6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2e8otp76oitb1jh1f4qm5u2spvk-w1t1q3s6x1e1x5d/router-577849c6f-7p6c6 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:13:50Z\n Ended: 2026-08-25T11:20:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig05qrvs2evs8a30l07oj4ofkolfp-l5a2z6l6l6v2z2w\", pod=\"router-766f6797fd-4t9pk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig05qrvs2evs8a30l07oj4ofkolfp-l5a2z6l6l6v2z2w/router-766f6797fd-4t9pk has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:22:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2sgfv21infum186h01mgb4tifri-j2m8e0t6e9r5w4s\", pod=\"ingress-operator-c798f4587-xnfgv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2sgfv21infum186h01mgb4tifri-j2m8e0t6e9r5w4s/ingress-operator-c798f4587-xnfgv has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:23:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiftlm8brlvli4n9drmv1vhs3oeth6-z8n3t9a8v2e7e7m\", pod=\"cluster-image-registry-operator-6f4bd7f774-wklr7\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiftlm8brlvli4n9drmv1vhs3oeth6-z8n3t9a8v2e7e7m/cluster-image-registry-operator-6f4bd7f774-wklr7 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:21:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2v4aa1s2gfru3f1uv3jlivc0jtd-p5t5x8w8z0r0g4h\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2v4aa1s2gfru3f1uv3jlivc0jtd-p5t5x8w8z0r0g4h/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:21:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2sgfv21infum186h01mgb4tifri-j2m8e0t6e9r5w4s\", pod=\"cluster-network-operator-85f5d57c7d-82nbp\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2sgfv21infum186h01mgb4tifri-j2m8e0t6e9r5w4s/cluster-network-operator-85f5d57c7d-82nbp has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:20:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2sgfv21infum186h01mgb4tifri-j2m8e0t6e9r5w4s\", pod=\"cluster-image-registry-operator-986f78f74-7d477\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2sgfv21infum186h01mgb4tifri-j2m8e0t6e9r5w4s/cluster-image-registry-operator-986f78f74-7d477 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:21:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdifubr6du1el2mlsfa6mhn6l93m64s-r9r3n5t4g5z1o6l\", pod=\"cluster-network-operator-66c6fd486f-97mb6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdifubr6du1el2mlsfa6mhn6l93m64s-r9r3n5t4g5z1o6l/cluster-network-operator-66c6fd486f-97mb6 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T11:14:50Z\n Ended: 2026-08-25T11:22:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig309ecpcs32nrdd7hs9n6tvobrvk-p6x2m3o7t1x5c7m\", pod=\"ingress-operator-785d88785-xhpgl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig309ecpcs32nrdd7hs9n6tvobrvk-p6x2m3o7t1x5c7m/ingress-operator-785d88785-xhpgl has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:23:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdifiu3kfspedp7bg2adod09p8tjs51-m4h6a7s1g6y0y7w\", pod=\"cluster-network-operator-867cf476f7-jhtbt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdifiu3kfspedp7bg2adod09p8tjs51-m4h6a7s1g6y0y7w/cluster-network-operator-867cf476f7-jhtbt has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:29:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig02iob6e2k6b50c08hc14r75pioo-ea-listwl9qrq\", pod=\"ingress-operator-6d5c4d9bf5-xzc5j\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig02iob6e2k6b50c08hc14r75pioo-ea-listwl9qrq/ingress-operator-6d5c4d9bf5-xzc5j has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:23:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig02iob6e2k6b50c08hc14r75pioo-ea-listwl9qrq\", pod=\"cluster-network-operator-854dbb4986-stn4n\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig02iob6e2k6b50c08hc14r75pioo-ea-listwl9qrq/cluster-network-operator-854dbb4986-stn4n has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:28:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig527d27t8t672b9d13c306pbqg6a-y0p1b3l1a2l2q3k\", pod=\"router-5658fb655-mr26f\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig527d27t8t672b9d13c306pbqg6a-y0p1b3l1a2l2q3k/router-5658fb655-mr26f has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:22:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig02iob6e2k6b50c08hc14r75pioo-ea-listwl9qrq\", pod=\"dns-operator-77874984f6-jqx8z\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig02iob6e2k6b50c08hc14r75pioo-ea-listwl9qrq/dns-operator-77874984f6-jqx8z has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:22:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdifiu3kfspedp7bg2adod09p8tjs51-m4h6a7s1g6y0y7w\", pod=\"ingress-operator-c7c7dc876-f9qx2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdifiu3kfspedp7bg2adod09p8tjs51-m4h6a7s1g6y0y7w/ingress-operator-c7c7dc876-f9qx2 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T11:16:50Z\n Ended: 2026-08-25T11:25:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig23l8p6umhg7m2pe7183ddad2jbq-p6f4c6v2i1p4t1k\", pod=\"router-69ff88c7b-xfcbr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig23l8p6umhg7m2pe7183ddad2jbq-p6f4c6v2i1p4t1k/router-69ff88c7b-xfcbr has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T11:17:49Z\n Ended: 2026-08-25T11:24:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2e8otp76oitb1jh1f4qm5u2spvk-w1t1q3s6x1e1x5d\", pod=\"openshift-apiserver-5876f5948f-2bqfp\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2e8otp76oitb1jh1f4qm5u2spvk-w1t1q3s6x1e1x5d/openshift-apiserver-5876f5948f-2bqfp has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T11:17:49Z\n Ended: 2026-08-25T11:25:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2hpum7e5n4s0jib2t987cajn7pc-v8b8k5t9b3l5k2k\", pod=\"ingress-operator-749cc6fb84-xrq6p\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2hpum7e5n4s0jib2t987cajn7pc-v8b8k5t9b3l5k2k/ingress-operator-749cc6fb84-xrq6p has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T11:17:50Z\n Ended: 2026-08-25T11:24:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2c4rftbtjejq4p3jigrj9s8vack-q2r5c0q7z3z1p6r\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2c4rftbtjejq4p3jigrj9s8vack-q2r5c0q7z3z1p6r/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-25T11:17:50Z\n Ended: 2026-08-25T11:24:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiftnonf8jdmjlr4jidiklg8rort12-e8t3p4z5l9d6i7m\", pod=\"multus-admission-controller-696ff77bdb-mfcs6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiftnonf8jdmjlr4jidiklg8rort12-e8t3p4z5l9d6i7m/multus-admission-controller-696ff77bdb-mfcs6 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-25T11:18:53Z\n Ended: 2026-08-25T11:24:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig23l8p6umhg7m2pe7183ddad2jbq-p6f4c6v2i1p4t1k\", pod=\"kube-apiserver-d85d65586-x7jxd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig23l8p6umhg7m2pe7183ddad2jbq-p6f4c6v2i1p4t1k/kube-apiserver-d85d65586-x7jxd has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-25T11:18:53Z\n Ended: 2026-08-25T11:30:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9005312-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig2e8otp76oitb1jh1f4qm5u2spvk-w1t1q3s6x1e1x5d\", pod=\"router-7999d5c7bc-g6gx8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig2e8otp76oitb1jh1f4qm5u2spvk-w1t1q3s6x1e1x5d/router-7999d5c7bc-g6gx8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ce6d643fc3b0d733d30c9abd8efed9f1b84a57bebbefe2c2c8e1aa7133735000","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"312f9db8ee161a9fc497742dfc7136dbb8ed546241173b0109c0ff6c461ac5e3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-p7lpzhts95vd/cluster-candidate-4-22-hlfnj9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ad040\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hlfnj9: failed waiting for cluster=\"cluster-candidate-4-22-hlfnj9\" in resourcegroup=\"rg-candidate-4-22-p7lpzhts95vd\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\",\n \"name\": \"18015e61-c049-41a2-809f-ad31666a4af5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.338382698Z\",\n \"endTime\": \"2026-08-25T11:20:48.26963606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-image-registry-operator, ingress-operator, kube-storage-version-migrator, catalog-operator, packageserver, certified-operators-catalog, csi-snapshot-controller-operator, community-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, redhat-operators-catalog, cluster-policy-controller, cluster-network-operator, machine-approver, dns-operator, olm-operator, openshift-apiserver, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-p7lpzhts95vd/cluster-candidate-4-22-hlfnj9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ad040\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hlfnj9: failed waiting for cluster=\"cluster-candidate-4-22-hlfnj9\" in resourcegroup=\"rg-candidate-4-22-p7lpzhts95vd\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\",\n \"name\": \"18015e61-c049-41a2-809f-ad31666a4af5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.338382698Z\",\n \"endTime\": \"2026-08-25T11:20:48.26963606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-image-registry-operator, ingress-operator, kube-storage-version-migrator, catalog-operator, packageserver, certified-operators-catalog, csi-snapshot-controller-operator, community-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, redhat-operators-catalog, cluster-policy-controller, cluster-network-operator, machine-approver, dns-operator, olm-operator, openshift-apiserver, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e367f2a3465740bc9b9da68a2f81e3714490aba74464e39ae8e5c3a1ae5bab94","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, kube-storage-version-migrator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8cba628d0e396bee5d06c282e9be947e9a625be75117b13627e941921db7beb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"96854033898edccbf2bb6b496287f357219f96280efefccded408cffb3a2f090","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nkwvwj\" for cluster \"cluster-candidate-5-0-nkwvwj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000179060\u003e: \n failed to create NodePool np-nkwvwj: failed waiting for nodepool=\"np-nkwvwj\" for cluster \"cluster-candidate-5-0-nkwvwj\" in resourcegroup=\"rg-candidate-5-0-qh5dlq8h25tn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02a0680c-d02a-4979-935f-a2788b9917ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02a0680c-d02a-4979-935f-a2788b9917ce\",\n \"name\": \"02a0680c-d02a-4979-935f-a2788b9917ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:20:40.33393292Z\",\n \"endTime\": \"2026-08-25T11:39:44.596654856Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nkwvwj\" for cluster \"cluster-candidate-5-0-nkwvwj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000179060\u003e: \n failed to create NodePool np-nkwvwj: failed waiting for nodepool=\"np-nkwvwj\" for cluster \"cluster-candidate-5-0-nkwvwj\" in resourcegroup=\"rg-candidate-5-0-qh5dlq8h25tn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02a0680c-d02a-4979-935f-a2788b9917ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02a0680c-d02a-4979-935f-a2788b9917ce\",\n \"name\": \"02a0680c-d02a-4979-935f-a2788b9917ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:20:40.33393292Z\",\n \"endTime\": \"2026-08-25T11:39:44.596654856Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f1ea0becb94441274dfde5eb9500c51e99c9d00da21d90f47ae2b444746d8e08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"2634ba4314b4b33fc2951e3d9e981c753c0c9fdc75af1b8135f694c4402837b6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af62e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j68lm294d7v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"name\": \"e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.830511566Z\",\n \"endTime\": \"2026-08-25T11:20:48.170023371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af62e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j68lm294d7v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"name\": \"e4433c13-3a16-45ba-88e7-ec043217acd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:40.830511566Z\",\n \"endTime\": \"2026-08-25T11:20:48.170023371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f63be4eca137ff9ea1d3e15142502577ac8278f05e03ac3a5880eeeb4ec9469f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"5ac2db27048619c0a83b9d739f995196c79b0762abf056bc200ed007d3557d7c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b7840\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tz75dw8glcvr\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83546c4c-4fc8-4a2c-89ac-1ab5e50bf205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83546c4c-4fc8-4a2c-89ac-1ab5e50bf205\",\n \"name\": \"83546c4c-4fc8-4a2c-89ac-1ab5e50bf205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:19:39.701946691Z\",\n \"endTime\": \"2026-08-25T11:38:44.54436442Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b7840\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tz75dw8glcvr\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83546c4c-4fc8-4a2c-89ac-1ab5e50bf205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83546c4c-4fc8-4a2c-89ac-1ab5e50bf205\",\n \"name\": \"83546c4c-4fc8-4a2c-89ac-1ab5e50bf205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:19:39.701946691Z\",\n \"endTime\": \"2026-08-25T11:38:44.54436442Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb76a0d9bacb43cd4deb9fba04567473de9a97d8d98e98762a8c3367356e4c38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","failed_at":"e2e","signature_id":"6586291399f5b3fd7dbb07efd25721a7c2283129efed74cec97d843274ba632c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bp8trx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d85000\u003e: \n failed to create HCP cluster cluster-zstream-5-0-bp8trx: failed waiting for cluster=\"cluster-zstream-5-0-bp8trx\" in resourcegroup=\"rg-zstream-upgrade-5-0-brkg7nwpr7jw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\",\n \"name\": \"3b253226-385a-4503-a738-5e57c00054ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.290629965Z\",\n \"endTime\": \"2026-08-25T11:20:27.735529124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bp8trx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d85000\u003e: \n failed to create HCP cluster cluster-zstream-5-0-bp8trx: failed waiting for cluster=\"cluster-zstream-5-0-bp8trx\" in resourcegroup=\"rg-zstream-upgrade-5-0-brkg7nwpr7jw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\",\n \"name\": \"3b253226-385a-4503-a738-5e57c00054ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:19.290629965Z\",\n \"endTime\": \"2026-08-25T11:20:27.735529124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":29,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","2fc8daa06dcd0810c6da5b7d738c6be1c97e759c090dbe32d9900028e598f859","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0371ff83ff8ae672d5d38be2f67ae6f726e275d3db1d866178e427376320fd8","a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","c97c7e10e9984956816f2978cee06c0347355fb43e2e18dbc518e10a50fd697c","da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","e367f2a3465740bc9b9da68a2f81e3714490aba74464e39ae8e5c3a1ae5bab94","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":29,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"open","pr_sha":"a095cb396dbd9be532d31b74ffc03a6b6bcd4cf9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:18:15Z"},"failed_at":["alert","e2e"],"failed_test_count":30,"occurrences":[{"row_id":"00d7c01fdd70c4bc545261f9dc8aa1d2e50ac0b7b5eb7eded189a7b02989e49f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"d7b472469e4b347e99e5eb839d0b4c4d6750f5bc580fe678d89b0781d5908848","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018140\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gvdtksqqw7j8\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3e1242e-a94b-425d-8b4d-fa06d2347c55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3e1242e-a94b-425d-8b4d-fa06d2347c55\",\n \"name\": \"f3e1242e-a94b-425d-8b4d-fa06d2347c55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:42.464677853Z\",\n \"endTime\": \"2026-08-25T11:20:47.7869157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018140\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gvdtksqqw7j8\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3e1242e-a94b-425d-8b4d-fa06d2347c55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3e1242e-a94b-425d-8b4d-fa06d2347c55\",\n \"name\": \"f3e1242e-a94b-425d-8b4d-fa06d2347c55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:42.464677853Z\",\n \"endTime\": \"2026-08-25T11:20:47.7869157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"013f1ea428a28e53cc54f34234ee2f11a44209854c6d142ba850f9ef7fdf97de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"58a3af70c95169637c5e6f6aa6a771a555147df7756a6c67c6c7a78311b59eee","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001178620\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-pp7257vrdlf5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64c0748d-e4ba-4941-92be-50ba2fb09d8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64c0748d-e4ba-4941-92be-50ba2fb09d8b\",\n \"name\": \"64c0748d-e4ba-4941-92be-50ba2fb09d8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:02.431917892Z\",\n \"endTime\": \"2026-08-25T11:21:07.397091958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001178620\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-pp7257vrdlf5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64c0748d-e4ba-4941-92be-50ba2fb09d8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64c0748d-e4ba-4941-92be-50ba2fb09d8b\",\n \"name\": \"64c0748d-e4ba-4941-92be-50ba2fb09d8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:02.431917892Z\",\n \"endTime\": \"2026-08-25T11:21:07.397091958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"10638375637d7f2d32617d1bbb7aef87f8d700fd782948c67702ce3cd6ba3c12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"alert","signature_id":"f9b1dbaf8f8300b20f00ab73fece95cb317fd0f48d3a396dbbe2c51c020ad097","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:04:14Z\n Ended: 2026-08-25T11:37:19Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2151936-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2151936-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"24f76fcec8782a18a0c0b08dc16796eac8ad38dd4d0f09917caa8fe2beaf3a67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"4aece071fa661e61fc57264421a76c9eac13620f5e30bdc56e63af5b1984f5a1","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000900480\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dgghghmcnzdm\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa686f08-f4cc-4a79-8daa-f55b889c2e5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa686f08-f4cc-4a79-8daa-f55b889c2e5b\",\n \"name\": \"aa686f08-f4cc-4a79-8daa-f55b889c2e5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.300970261Z\",\n \"endTime\": \"2026-08-25T11:20:57.373035491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000900480\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dgghghmcnzdm\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa686f08-f4cc-4a79-8daa-f55b889c2e5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa686f08-f4cc-4a79-8daa-f55b889c2e5b\",\n \"name\": \"aa686f08-f4cc-4a79-8daa-f55b889c2e5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.300970261Z\",\n \"endTime\": \"2026-08-25T11:20:57.373035491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2c425ff594460ee3b0456c620d7951a81d61abc6904cd5029976cf39e8f9fbd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"c4caf661a44dbe4ab9a819e2a4f4d4a40e4840ddd13380deac01741d243deeb9","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000855700\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-8spmvpshpxg4\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e884ed9-970f-40fe-b266-642f03ffe736\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e884ed9-970f-40fe-b266-642f03ffe736\",\n \"name\": \"8e884ed9-970f-40fe-b266-642f03ffe736\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.700430635Z\",\n \"endTime\": \"2026-08-25T11:20:57.397456607Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000855700\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-8spmvpshpxg4\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e884ed9-970f-40fe-b266-642f03ffe736\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e884ed9-970f-40fe-b266-642f03ffe736\",\n \"name\": \"8e884ed9-970f-40fe-b266-642f03ffe736\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.700430635Z\",\n \"endTime\": \"2026-08-25T11:20:57.397456607Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"31e69166f3fed9a25a761e818c3914eef1351f2fa878861b7a21bf62a48c7db7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"c5290fb9c57d434bec526c46d513016dd6789c9d085ab04d2a242efeb095b470","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02900\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-sn9l5gh5sgnq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"name\": \"77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:02.408692636Z\",\n \"endTime\": \"2026-08-25T11:21:07.333560396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02900\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-sn9l5gh5sgnq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"name\": \"77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:02.408692636Z\",\n \"endTime\": \"2026-08-25T11:21:07.333560396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"691000237ed68ee2bea1eca4972f51b642a5c69ce54dd17b387ef5ef88b01453","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"372a7d3053c3df0219a08d5d38a07dd61507a5b6cf40d9ec35467e015736d59f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"d17d79e8ac084d555f983d2abb36810032fbe5d1d3056157e89dc068b55ec851","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-fhgqh26vqnp9/cluster-candidate-4-22-lw4fjm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a02400\u003e: \n failed to create HCP cluster cluster-candidate-4-22-lw4fjm: failed waiting for cluster=\"cluster-candidate-4-22-lw4fjm\" in resourcegroup=\"rg-candidate-4-22-fhgqh26vqnp9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99c474d9-a201-4fec-b07b-8d9819fe8425\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99c474d9-a201-4fec-b07b-8d9819fe8425\",\n \"name\": \"99c474d9-a201-4fec-b07b-8d9819fe8425\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.421892504Z\",\n \"endTime\": \"2026-08-25T11:20:57.269333787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-storage-operator, catalog-operator, certified-operators-catalog, olm-operator, packageserver, hosted-cluster-config-operator, ingress-operator, community-operators-catalog, csi-snapshot-controller-operator, redhat-operators-catalog, cluster-network-operator, cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-fhgqh26vqnp9/cluster-candidate-4-22-lw4fjm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a02400\u003e: \n failed to create HCP cluster cluster-candidate-4-22-lw4fjm: failed waiting for cluster=\"cluster-candidate-4-22-lw4fjm\" in resourcegroup=\"rg-candidate-4-22-fhgqh26vqnp9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99c474d9-a201-4fec-b07b-8d9819fe8425\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99c474d9-a201-4fec-b07b-8d9819fe8425\",\n \"name\": \"99c474d9-a201-4fec-b07b-8d9819fe8425\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.421892504Z\",\n \"endTime\": \"2026-08-25T11:20:57.269333787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-storage-operator, catalog-operator, certified-operators-catalog, olm-operator, packageserver, hosted-cluster-config-operator, ingress-operator, community-operators-catalog, csi-snapshot-controller-operator, redhat-operators-catalog, cluster-network-operator, cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3cdeb09912038b499138071a5026a1113b603df6084154b13d57bdad04ef0585","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"09dd934fdb303c274c99f6ad4ed42588a7c9f786ef23d8ba1c2d65fa831cb2e2","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014868a0\u003e: \n failed to create HCP cluster np-dg-minor-6lfgc2: failed waiting for cluster=\"np-dg-minor-6lfgc2\" in resourcegroup=\"rg-np-dg-minor-6lfgc2-lwtz9m4rnlh9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\",\n \"name\": \"169a9726-cf09-492c-bf9a-53259660198c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.246951809Z\",\n \"endTime\": \"2026-08-25T11:20:57.403791086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014868a0\u003e: \n failed to create HCP cluster np-dg-minor-6lfgc2: failed waiting for cluster=\"np-dg-minor-6lfgc2\" in resourcegroup=\"rg-np-dg-minor-6lfgc2-lwtz9m4rnlh9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\",\n \"name\": \"169a9726-cf09-492c-bf9a-53259660198c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.246951809Z\",\n \"endTime\": \"2026-08-25T11:20:57.403791086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"66285e3570a1d3a91819718028ead75ff34527595051ad0083e72a103f62194a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"401d4ff34d1c5d16fbeb9499c866198ede83beee3963c9d6999dd46962aa525c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"9a6939f0032bf8fdc09bb65ca1155a9ec01d12b02a56acd5a25adfca6beb61f3","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014d4ba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-djpx8vzhvdqj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"name\": \"a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.971395174Z\",\n \"endTime\": \"2026-08-25T11:20:37.411972437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, dns-operator, ingress-operator, ignition-server, olm-collect-profiles, certified-operators-catalog, kube-controller-manager, openshift-apiserver, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, ignition-server-proxy, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, machine-approver, packageserver, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014d4ba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-djpx8vzhvdqj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"name\": \"a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.971395174Z\",\n \"endTime\": \"2026-08-25T11:20:37.411972437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, dns-operator, ingress-operator, ignition-server, olm-collect-profiles, certified-operators-catalog, kube-controller-manager, openshift-apiserver, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, ignition-server-proxy, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, machine-approver, packageserver, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42a3dac5e75c0c26d4ee4023b4684411f559f06bf6a2d4f2290251ab07142923","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"3397e3c177e331b0b3c56e3aca57d39b4697fcc3c0632de87b50427da89cf9dd","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-mbbxgn with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbe500\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mbbxgn: failed waiting for cluster=\"np-version-upgrade-cluster-mbbxgn\" in resourcegroup=\"rg-np-version-upgrade-mbbxgn-djj4c2xjbmdj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60437c45-38ab-47e8-b7f9-9c016655bbda\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60437c45-38ab-47e8-b7f9-9c016655bbda\",\n \"name\": \"60437c45-38ab-47e8-b7f9-9c016655bbda\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:42.841852986Z\",\n \"endTime\": \"2026-08-25T11:20:47.887806017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-mbbxgn with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbe500\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mbbxgn: failed waiting for cluster=\"np-version-upgrade-cluster-mbbxgn\" in resourcegroup=\"rg-np-version-upgrade-mbbxgn-djj4c2xjbmdj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60437c45-38ab-47e8-b7f9-9c016655bbda\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60437c45-38ab-47e8-b7f9-9c016655bbda\",\n \"name\": \"60437c45-38ab-47e8-b7f9-9c016655bbda\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:42.841852986Z\",\n \"endTime\": \"2026-08-25T11:20:47.887806017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"478fdcb94a28b9404709be1faa097ba0d4388c5950b8a264b49996a99b6d461c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"32446f0653d184d03231d5fec01e7294672fb079eed4f9359b944cae9896c2fc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c70e60\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-j2h7qqprw2cq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d441ebfa-adad-47e3-810c-f2ff28ffd4b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d441ebfa-adad-47e3-810c-f2ff28ffd4b3\",\n \"name\": \"d441ebfa-adad-47e3-810c-f2ff28ffd4b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:15.897556457Z\",\n \"endTime\": \"2026-08-25T11:23:17.388317431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c70e60\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-j2h7qqprw2cq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d441ebfa-adad-47e3-810c-f2ff28ffd4b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d441ebfa-adad-47e3-810c-f2ff28ffd4b3\",\n \"name\": \"d441ebfa-adad-47e3-810c-f2ff28ffd4b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:15.897556457Z\",\n \"endTime\": \"2026-08-25T11:23:17.388317431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b7658b90551e3fd54bb86116d76ca3954835289414d0635c6239ea1972c7f8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"dbfac9a592c7d3b659769802f66b5c2ea426b38b8a462e72863bc31b516fbcfb","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013da220\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hk9dqp59j9m5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"name\": \"cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.161881545Z\",\n \"endTime\": \"2026-08-25T11:20:57.408325813Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013da220\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hk9dqp59j9m5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"name\": \"cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.161881545Z\",\n \"endTime\": \"2026-08-25T11:20:57.408325813Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"01b6bec7cc22825d4ef87b95818e501dbd62c6d195e057bf40cb1fb4d00f77e0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c6d14500989e640fd92043630b0ff474df0fa629f239dd5224df6cad09b8409","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"3ab9e7c7d4f2d79e65f9b8a56c90ee66becb35144ac46e78ca458d7f63faa98d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-rc5jrpz77lgk/cluster-candidate-5-0-k8xlxd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001060a40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-k8xlxd: failed waiting for cluster=\"cluster-candidate-5-0-k8xlxd\" in resourcegroup=\"rg-candidate-5-0-rc5jrpz77lgk\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"name\": \"223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.648244013Z\",\n \"endTime\": \"2026-08-25T11:20:37.341589368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-rc5jrpz77lgk/cluster-candidate-5-0-k8xlxd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001060a40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-k8xlxd: failed waiting for cluster=\"cluster-candidate-5-0-k8xlxd\" in resourcegroup=\"rg-candidate-5-0-rc5jrpz77lgk\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"name\": \"223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:32.648244013Z\",\n \"endTime\": \"2026-08-25T11:20:37.341589368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5d1725423ee4b212bccb0d13858a78dd47769db5d794223ea2e26d6df7c4f5ad","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"50995b622d9dbef2104c2e8387310d327ff654fb96b6bd9b0f4e6eb97f41877e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"80ba9276ccee66ce5f15246d1c46ed27a227c963837a0639556a2c98e2273b4b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706220\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdgbq5vc5q77\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"name\": \"e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:45.146044639Z\",\n \"endTime\": \"2026-08-25T11:21:47.576163129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706220\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdgbq5vc5q77\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"name\": \"e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:45.146044639Z\",\n \"endTime\": \"2026-08-25T11:21:47.576163129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2256dab0b9b860af42d8774bc9e3bdf3a75814dca1080b715190d3d8ab7a2eb9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60f98c567d752594a443bb06fed3444d77f3d0772e00a675b914254ed0b1c7d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"d170f04f7cb6344abd769b118fc7f073e431f9b518426774c88ff7532f8e723d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a2260\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-kp69jsvl6hm4\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6e6c7c1-86af-4757-abbe-ed454d6ac3b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6e6c7c1-86af-4757-abbe-ed454d6ac3b8\",\n \"name\": \"b6e6c7c1-86af-4757-abbe-ed454d6ac3b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:44.186307399Z\",\n \"endTime\": \"2026-08-25T11:20:47.798437893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a2260\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-kp69jsvl6hm4\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6e6c7c1-86af-4757-abbe-ed454d6ac3b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6e6c7c1-86af-4757-abbe-ed454d6ac3b8\",\n \"name\": \"b6e6c7c1-86af-4757-abbe-ed454d6ac3b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:44.186307399Z\",\n \"endTime\": \"2026-08-25T11:20:47.798437893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6113a7f2e643aa73f1be8f6f2e5692cc7a8fd313903c7e15e354ddd61ff14cde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"25be28244289ac46a2d410d588e4df19d0867d5c662db6e3ba488825169d0ea1","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00062e1c0\u003e: \n failed to create HCP cluster ea-listqqp748: failed waiting for cluster=\"ea-listqqp748\" in resourcegroup=\"ea-list-t4fnk5lsvq88\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c1a2b240-6346-4e98-b4bb-870288292b46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c1a2b240-6346-4e98-b4bb-870288292b46\",\n \"name\": \"c1a2b240-6346-4e98-b4bb-870288292b46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:13.223545992Z\",\n \"endTime\": \"2026-08-25T11:21:17.3486915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, csi-snapshot-controller-operator, openshift-controller-manager, olm-operator, cluster-image-registry-operator, catalog-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-version-operator, packageserver, hosted-cluster-config-operator, cluster-node-tuning-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00062e1c0\u003e: \n failed to create HCP cluster ea-listqqp748: failed waiting for cluster=\"ea-listqqp748\" in resourcegroup=\"ea-list-t4fnk5lsvq88\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c1a2b240-6346-4e98-b4bb-870288292b46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c1a2b240-6346-4e98-b4bb-870288292b46\",\n \"name\": \"c1a2b240-6346-4e98-b4bb-870288292b46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:13.223545992Z\",\n \"endTime\": \"2026-08-25T11:21:17.3486915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, csi-snapshot-controller-operator, openshift-controller-manager, olm-operator, cluster-image-registry-operator, catalog-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-version-operator, packageserver, hosted-cluster-config-operator, cluster-node-tuning-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"78c7f97c684fb8777690d09c7b9cbfe2a80f2cf547b600446c0e04c872ae1fff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"a0b46f2049c857f9b50193bfa943a46b002ac4931192ad3d996324be47f034d8","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea1740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jm4vsb7vn4sd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"name\": \"afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:22.824480944Z\",\n \"endTime\": \"2026-08-25T11:22:27.407570836Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea1740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jm4vsb7vn4sd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"name\": \"afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:22.824480944Z\",\n \"endTime\": \"2026-08-25T11:22:27.407570836Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db4713748d0354976aeef8212e3a9f4a93e3ecf500342fbbcb1fbd6a21579f7d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"81ef26680bcd5ad7f72b06d03e002df6fe28b8b6daae06f78e40edb3e46c786d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"f45e48a3b1e3da0b3b86c68ae16171bf54eb097156d7da3966f4506df7b9b4dc","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097b680\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wgz7h9k2skjd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"name\": \"a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19619894Z\",\n \"endTime\": \"2026-08-25T11:20:57.271055091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097b680\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wgz7h9k2skjd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"name\": \"a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19619894Z\",\n \"endTime\": \"2026-08-25T11:20:57.271055091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bebd56cb7971d5cbd2ba0627c63fe9925ffa3a6799fdb1d98a37f3159a2c38aa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82ed956956bd12856af756924b09c1366f19806d060cab8a369ab11db8c6c08f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"a43867351bcfa3c50d9cc6d93c93e014ba943a32a17885c7a13ae5f00205adb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"alert","signature_id":"ab1ad628fd91e461795e6817e8295d83b1b66efac12206305c9fb1793f4943a9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 133 time(s) (48 unknown, 85 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:12:39Z\n Ended: 2026-08-25T11:19:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig5p67f664gdu1m2g9bl8mcvifgd7-b0n9x4p4j4r7o9p\", pod=\"cluster-network-operator-66bccc674-f9jd2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig5p67f664gdu1m2g9bl8mcvifgd7-b0n9x4p4j4r7o9p/cluster-network-operator-66bccc674-f9jd2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:12:39Z\n Ended: 2026-08-25T11:18:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigdkfodll0bdu2u3blkkpq2isdp01-ea-listqqp748\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigdkfodll0bdu2u3blkkpq2isdp01-ea-listqqp748/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig34vqir757268i5b4b1ol83t92ql-f3e5u5i7x7y2v3b\", pod=\"dns-operator-68d4b6f668-l9ddx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig34vqir757268i5b4b1ol83t92ql-f3e5u5i7x7y2v3b/dns-operator-68d4b6f668-l9ddx has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8dm0oe272dqghvft882nvmv2jtt-cilium-cl\", pod=\"cluster-network-operator-5765bb4777-txdkz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8dm0oe272dqghvft882nvmv2jtt-cilium-cl/cluster-network-operator-5765bb4777-txdkz has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:20:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8cujb2061821rrqkcioldosjt2v-g9n3s2f2k8n1t7q\", pod=\"openshift-apiserver-5576f4f4d4-97286\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8cujb2061821rrqkcioldosjt2v-g9n3s2f2k8n1t7q/openshift-apiserver-5576f4f4d4-97286 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiftinbamjaq0j5tbanqtu5nns4e4p-t4b7e0d9q4i1q1n\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiftinbamjaq0j5tbanqtu5nns4e4p-t4b7e0d9q4i1q1n/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig38ddbi8bj330ci2r2kcpqdbjvuu-v1e7o9p8l0z2o9s\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig38ddbi8bj330ci2r2kcpqdbjvuu-v1e7o9p8l0z2o9s/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:20:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigarcqk1jc7jjqkmj3beg8b4voupr-o5u3a3y3t1g7p9i\", pod=\"openshift-apiserver-7cdd6bc555-5hzp7\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigarcqk1jc7jjqkmj3beg8b4voupr-o5u3a3y3t1g7p9i/openshift-apiserver-7cdd6bc555-5hzp7 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig819rv81qsapl7h6p95729fbi8l7-u8g6a6c8k9v4d4v\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig819rv81qsapl7h6p95729fbi8l7-u8g6a6c8k9v4d4v/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:20:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster\", pod=\"ignition-server-proxy-5bb6484f8f-4797l\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster/ignition-server-proxy-5bb6484f8f-4797l has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:20:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig84uregpbaeku352dmjngjarq0l3-z2z7s9l1f1s1q8t\", pod=\"ignition-server-proxy-5d884fd766-zn7rk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig84uregpbaeku352dmjngjarq0l3-z2z7s9l1f1s1q8t/ignition-server-proxy-5d884fd766-zn7rk has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig34vqir757268i5b4b1ol83t92ql-f3e5u5i7x7y2v3b\", pod=\"cluster-node-tuning-operator-688f489b78-msrr5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig34vqir757268i5b4b1ol83t92ql-f3e5u5i7x7y2v3b/cluster-node-tuning-operator-688f489b78-msrr5 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T11:15:40Z\n Ended: 2026-08-25T11:21:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8ts4tcq3j83brb5tgspj33kr4jf-s1f3w3v0x3p9g4v\", pod=\"multus-admission-controller-84f697989b-vlk62\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8ts4tcq3j83brb5tgspj33kr4jf-s1f3w3v0x3p9g4v/multus-admission-controller-84f697989b-vlk62 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T11:15:40Z\n Ended: 2026-08-25T11:21:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig644i6o0am3lvqru4vdu72s9o4kl-w4i4n4y2w1o0u1r\", pod=\"ingress-operator-94d4b54f6-jccsg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig644i6o0am3lvqru4vdu72s9o4kl-w4i4n4y2w1o0u1r/ingress-operator-94d4b54f6-jccsg has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T11:15:40Z\n Ended: 2026-08-25T11:21:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiggk1koknr2fnqm2kiv8hqodjkdro-j5z4w7m2q6y2f3e\", pod=\"openshift-controller-manager-787d4c48d9-zmvmp\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiggk1koknr2fnqm2kiv8hqodjkdro-j5z4w7m2q6y2f3e/openshift-controller-manager-787d4c48d9-zmvmp has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T11:15:42Z\n Ended: 2026-08-25T11:21:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8r5a6pfu7k927miu129sk7s4ptu-ephemeral-disk\", pod=\"kube-apiserver-7bcb44bd57-4lfgq\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8r5a6pfu7k927miu129sk7s4ptu-ephemeral-disk/kube-apiserver-7bcb44bd57-4lfgq has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T11:15:42Z\n Ended: 2026-08-25T11:21:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig644i6o0am3lvqru4vdu72s9o4kl-w4i4n4y2w1o0u1r\", pod=\"cluster-network-operator-6969665b4-vvwq5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig644i6o0am3lvqru4vdu72s9o4kl-w4i4n4y2w1o0u1r/cluster-network-operator-6969665b4-vvwq5 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T11:15:43Z\n Ended: 2026-08-25T11:29:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8cujb2061821rrqkcioldosjt2v-g9n3s2f2k8n1t7q\", pod=\"router-55674ccc44-pvvww\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8cujb2061821rrqkcioldosjt2v-g9n3s2f2k8n1t7q/router-55674ccc44-pvvww has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T11:16:40Z\n Ended: 2026-08-25T11:34:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigaotum572v7tp065fldok2ga0duj-v20260901\", pod=\"router-dc6b8f666-nfl8q\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigaotum572v7tp065fldok2ga0duj-v20260901/router-dc6b8f666-nfl8q has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-25T11:16:41Z\n Ended: 2026-08-25T11:23:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigapoi3212giljien9ra888at6kjb-sre-hcp-cluster\", pod=\"ingress-operator-788df6f5bf-hkq6p\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigapoi3212giljien9ra888at6kjb-sre-hcp-cluster/ingress-operator-788df6f5bf-hkq6p has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-25T11:16:41Z\n Ended: 2026-08-25T11:29:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig3erhiqelo66rmekapehp0qpnvp5-u7i2f6t6c4k3k0m\", pod=\"router-85b5dfdd44-98b8g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig3erhiqelo66rmekapehp0qpnvp5-u7i2f6t6c4k3k0m/router-85b5dfdd44-98b8g has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-25T11:16:41Z\n Ended: 2026-08-25T11:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig38eme5vr4f34ngc821ehqehdv26-h3q7l3s0e8q5p1m\", pod=\"azure-file-csi-driver-controller-6445ccdb57-69cxn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig38eme5vr4f34ngc821ehqehdv26-h3q7l3s0e8q5p1m/azure-file-csi-driver-controller-6445ccdb57-69cxn has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-08-25T11:16:41Z\n Ended: 2026-08-25T11:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig5v5uqol5cmac2j0cndb1s9cu2tq-i8j6t1e4v0q3h0a\", pod=\"cluster-network-operator-86c4c9bf7c-2qtjg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig5v5uqol5cmac2j0cndb1s9cu2tq-i8j6t1e4v0q3h0a/cluster-network-operator-86c4c9bf7c-2qtjg has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-08-25T11:17:41Z\n Ended: 2026-08-25T11:24:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig5vcmg991o3cj5ur28dqke5597h9-h5a5a9y0a8v5l4j\", pod=\"cluster-network-operator-67d5d5b7d6-96ptn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig5vcmg991o3cj5ur28dqke5597h9-h5a5a9y0a8v5l4j/cluster-network-operator-67d5d5b7d6-96ptn has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster\", pod=\"ingress-operator-68d4cc6849-tq4gb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster/ingress-operator-68d4cc6849-tq4gb has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig3erhiqelo66rmekapehp0qpnvp5-u7i2f6t6c4k3k0m\", pod=\"machine-approver-64b9b94cb4-ltnvw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig3erhiqelo66rmekapehp0qpnvp5-u7i2f6t6c4k3k0m/machine-approver-64b9b94cb4-ltnvw has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:31:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig862dk1tnff6ftg9i4f840029fum-j6u3d4g4y5d4a5y\", pod=\"router-dc7df8b4b-mb5bt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig862dk1tnff6ftg9i4f840029fum-j6u3d4g4y5d4a5y/router-dc7df8b4b-mb5bt has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster\", pod=\"cluster-image-registry-operator-57d6dcf4d4-sp59g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster/cluster-image-registry-operator-57d6dcf4d4-sp59g has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q\", pod=\"ingress-operator-57d759d85c-cjhvx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q/ingress-operator-57d759d85c-cjhvx has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q\", pod=\"dns-operator-55657bf974-bnrbh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q/dns-operator-55657bf974-bnrbh has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig608melq35so7gm1quh13emo8muf-q7j3f1e7u4f3t7q\", pod=\"cluster-network-operator-7bfd8bf4bc-2wwlx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig608melq35so7gm1quh13emo8muf-q7j3f1e7u4f3t7q/cluster-network-operator-7bfd8bf4bc-2wwlx has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q\", pod=\"cluster-network-operator-6f59dc5797-8l6qs\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q/cluster-network-operator-6f59dc5797-8l6qs has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster\", pod=\"openshift-controller-manager-758df4ffd7-r9m45\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tmoe0ak1l116rq7u8af74ia3e6-oidc-wi-cluster/openshift-controller-manager-758df4ffd7-r9m45 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q\", pod=\"dns-operator-7d75777d6b-v8zxj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigana8st2onnrv326q7o7860hbrtm-z2n7y5t7o2w2p0q/dns-operator-7d75777d6b-v8zxj has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:31:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig38eme5vr4f34ngc821ehqehdv26-h3q7l3s0e8q5p1m\", pod=\"azure-disk-csi-driver-controller-fcf46799f-qp4v6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig38eme5vr4f34ngc821ehqehdv26-h3q7l3s0e8q5p1m/azure-disk-csi-driver-controller-fcf46799f-qp4v6 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:26:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p\", pod=\"csi-snapshot-controller-operator-79cfccd995-9zlzj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p/csi-snapshot-controller-operator-79cfccd995-9zlzj has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8mc00gu1gpaopm9r03966vng1f9-d1b4g5o4p9w4n0l\", pod=\"multus-admission-controller-79d7ccc4c8-rrxfn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8mc00gu1gpaopm9r03966vng1f9-d1b4g5o4p9w4n0l/multus-admission-controller-79d7ccc4c8-rrxfn has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig5v5uqol5cmac2j0cndb1s9cu2tq-i8j6t1e4v0q3h0a\", pod=\"azure-file-csi-driver-operator-678d98dc57-zb8kj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig5v5uqol5cmac2j0cndb1s9cu2tq-i8j6t1e4v0q3h0a/azure-file-csi-driver-operator-678d98dc57-zb8kj has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihc38ad9t2fqbgqsetulikg35kj4d-agg-cluster\", pod=\"openshift-apiserver-6b856db4fd-t2hdh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihc38ad9t2fqbgqsetulikg35kj4d-agg-cluster/openshift-apiserver-6b856db4fd-t2hdh has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig5oudr0ppt5j63nvg2lathk0pvcd-b9e4d1i8o2d4s4b\", pod=\"openshift-controller-manager-547df955b4-g2v7j\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig5oudr0ppt5j63nvg2lathk0pvcd-b9e4d1i8o2d4s4b/openshift-controller-manager-547df955b4-g2v7j has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:29:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p\", pod=\"ingress-operator-5c7c7d9d95-dljws\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p/ingress-operator-5c7c7d9d95-dljws has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigapoi3212giljien9ra888at6kjb-sre-hcp-cluster\", pod=\"azure-disk-csi-driver-operator-5d7595c449-nrwjg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigapoi3212giljien9ra888at6kjb-sre-hcp-cluster/azure-disk-csi-driver-operator-5d7595c449-nrwjg has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8mc00gu1gpaopm9r03966vng1f9-d1b4g5o4p9w4n0l\", pod=\"multus-admission-controller-79d7ccc4c8-w86gx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8mc00gu1gpaopm9r03966vng1f9-d1b4g5o4p9w4n0l/multus-admission-controller-79d7ccc4c8-w86gx has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-08-25T11:20:40Z\n Ended: 2026-08-25T11:26:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigau4l6s2auv43thna4dh8btjadmo-private-ingress\", pod=\"machine-approver-7ccbff6cdc-qhdv8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigau4l6s2auv43thna4dh8btjadmo-private-ingress/machine-approver-7ccbff6cdc-qhdv8 has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-08-25T11:20:40Z\n Ended: 2026-08-25T11:28:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p\", pod=\"cluster-network-operator-5d8f449bdd-skx5k\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p/cluster-network-operator-5d8f449bdd-skx5k has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-08-25T11:20:40Z\n Ended: 2026-08-25T11:26:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p\", pod=\"cluster-storage-operator-c9f4b55c6-sgsj5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p/cluster-storage-operator-c9f4b55c6-sgsj5 has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-08-25T11:20:40Z\n Ended: 2026-08-25T11:26:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p\", pod=\"dns-operator-58f6959675-w8npm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p/dns-operator-58f6959675-w8npm has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-08-25T11:20:40Z\n Ended: 2026-08-25T11:26:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2151936-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p\", pod=\"cluster-version-operator-64767fc9fc-j6c29\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig8tjgu9c5er3bs4kdcuvmp48h9bj-w6j2p6x4l5f8m1p/cluster-version-operator-64767fc9fc-j6c29 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"a99f73f98a880f51914c8048a06bc45395a4c26b3ab727f0e89add447e375ebd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"6cfd6ef557fef8c9d185efe4042060278e1497ede40310252790075002d9aac9","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--mdtqq6tfggzt/cluster-hs-pre-5md8sn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138c380\u003e: \n failed to create HCP cluster cluster-hs-pre-5md8sn: failed waiting for cluster=\"cluster-hs-pre-5md8sn\" in resourcegroup=\"rg-hs-pre--mdtqq6tfggzt\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\",\n \"name\": \"faa7138f-d176-4225-9189-5771c1682daa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.650546675Z\",\n \"endTime\": \"2026-08-25T11:20:57.345950087Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, redhat-marketplace-catalog, cluster-policy-controller, azure-cloud-controller-manager, cluster-image-registry-operator, certified-operators-catalog, packageserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, cluster-node-tuning-operator, openshift-apiserver, catalog-operator, community-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--mdtqq6tfggzt/cluster-hs-pre-5md8sn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138c380\u003e: \n failed to create HCP cluster cluster-hs-pre-5md8sn: failed waiting for cluster=\"cluster-hs-pre-5md8sn\" in resourcegroup=\"rg-hs-pre--mdtqq6tfggzt\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\",\n \"name\": \"faa7138f-d176-4225-9189-5771c1682daa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.650546675Z\",\n \"endTime\": \"2026-08-25T11:20:57.345950087Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, redhat-marketplace-catalog, cluster-policy-controller, azure-cloud-controller-manager, cluster-image-registry-operator, certified-operators-catalog, packageserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, cluster-node-tuning-operator, openshift-apiserver, catalog-operator, community-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"04ab38fb4caffee001686eb32195645f728f8555d294ffa19de0a9f505223827","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aac5d8438dad36d60819076b560199a437d666c06563899286d53ea6f6f507b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"18cefbe16bc079f2ead9cb47f111db79d94e08b882e7b91731e46458212a977b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: failed to load rendered config — is RENDERED_CONFIG set?\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eafac0\u003e: \n failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: failed to load rendered config — is RENDERED_CONFIG set?\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eafac0\u003e: \n failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95452bdc0ded378689e5f2c12b2d45cbf916f7a4e0676d618a92d0f3d8778ac4","failure_pattern":"failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory","search_query":"failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory"}},{"row_id":"b821157e7a6b86dc13fd3c24b502da77135ca68273b4701b75988c2b9acf15d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"effe32de00004a904c211837be3300222c5b0de9f1d089f9996c139580d10677","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5vxfzn with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005eed00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-5vxfzn: failed waiting for cluster=\"np-version-upgrade-cluster-5vxfzn\" in resourcegroup=\"rg-np-version-upgrade-5vxfzn-7svm57fp7wk7\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"name\": \"1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.133785755Z\",\n \"endTime\": \"2026-08-25T11:20:57.201457228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5vxfzn with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005eed00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-5vxfzn: failed waiting for cluster=\"np-version-upgrade-cluster-5vxfzn\" in resourcegroup=\"rg-np-version-upgrade-5vxfzn-7svm57fp7wk7\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"name\": \"1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:54.133785755Z\",\n \"endTime\": \"2026-08-25T11:20:57.201457228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c6ad338a9a82fdd269e2e826c9d43094ffe24f6b4e773ddce70975cc9f8e5127","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"25b082e6f3fccd0eeb0248caee3439349733372ff0b9b1c11e1a4ee3a27cb02e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14300\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nkvt2c5pdsrv\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecd5f2d-dd8e-444c-9b57-369de1b270b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecd5f2d-dd8e-444c-9b57-369de1b270b8\",\n \"name\": \"5ecd5f2d-dd8e-444c-9b57-369de1b270b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.287776651Z\",\n \"endTime\": \"2026-08-25T11:20:57.385655782Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, olm-operator, hosted-cluster-config-operator, cluster-network-operator, packageserver, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, catalog-operator, community-operators-catalog, ingress-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14300\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nkvt2c5pdsrv\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecd5f2d-dd8e-444c-9b57-369de1b270b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecd5f2d-dd8e-444c-9b57-369de1b270b8\",\n \"name\": \"5ecd5f2d-dd8e-444c-9b57-369de1b270b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:53.287776651Z\",\n \"endTime\": \"2026-08-25T11:20:57.385655782Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, olm-operator, hosted-cluster-config-operator, cluster-network-operator, packageserver, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, catalog-operator, community-operators-catalog, ingress-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cbf39efea6740bf11fdb7989ef9f17fff6a1c145b6b3f5786086e2aa9ad7574a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"alert","signature_id":"8e7a90f4497ef1b51e90f2043b6517b8c68d8bfd66277c1d9bf290617a22ad90","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:51:56Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j2151936-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2151936/providers/microsoft.monitor/accounts/services-j2151936\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"de498dd64aa65f0e5eed253390fc6b1408c18e817fd2e647380f61069006e051","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"6aa2e7ee830b9fa0230c736bace065dc0ed283e1350606e8f5015746076bf57f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-psp25pblz7q9/cluster-candidate-4-20-5xxwvr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000947020\u003e: \n failed to create HCP cluster cluster-candidate-4-20-5xxwvr: failed waiting for cluster=\"cluster-candidate-4-20-5xxwvr\" in resourcegroup=\"rg-candidate-4-20-psp25pblz7q9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac623cd0-8df5-45d0-b2af-bc97016cb511\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac623cd0-8df5-45d0-b2af-bc97016cb511\",\n \"name\": \"ac623cd0-8df5-45d0-b2af-bc97016cb511\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.648184474Z\",\n \"endTime\": \"2026-08-25T11:20:57.398865961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-psp25pblz7q9/cluster-candidate-4-20-5xxwvr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000947020\u003e: \n failed to create HCP cluster cluster-candidate-4-20-5xxwvr: failed waiting for cluster=\"cluster-candidate-4-20-5xxwvr\" in resourcegroup=\"rg-candidate-4-20-psp25pblz7q9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac623cd0-8df5-45d0-b2af-bc97016cb511\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ac623cd0-8df5-45d0-b2af-bc97016cb511\",\n \"name\": \"ac623cd0-8df5-45d0-b2af-bc97016cb511\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.648184474Z\",\n \"endTime\": \"2026-08-25T11:20:57.398865961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e00f02600db63a448ceb3bf7bf526eb1a45d97f00bd8b81047073db30a15dfa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"431ff7e575eec123143334467cdb2866bc6f04d93674992976852f19ccc99809","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-qs7l5q\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016408c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-qs7l5q: failed waiting for cluster=\"cp-ystream-upgrade-4-22-qs7l5q\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sn26t59hv7kg\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e7b354b0-edaa-4c54-8a45-d3577720bedd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e7b354b0-edaa-4c54-8a45-d3577720bedd\",\n \"name\": \"e7b354b0-edaa-4c54-8a45-d3577720bedd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:33.430099239Z\",\n \"endTime\": \"2026-08-25T11:20:37.33551722Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-qs7l5q\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016408c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-qs7l5q: failed waiting for cluster=\"cp-ystream-upgrade-4-22-qs7l5q\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sn26t59hv7kg\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e7b354b0-edaa-4c54-8a45-d3577720bedd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e7b354b0-edaa-4c54-8a45-d3577720bedd\",\n \"name\": \"e7b354b0-edaa-4c54-8a45-d3577720bedd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:33.430099239Z\",\n \"endTime\": \"2026-08-25T11:20:37.33551722Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e11f98a6823ea6f5b31f1e1bc61a08b18e0ccf43000742c96e77357c42ac6549","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"d5ffa576330ff4e403d279cd3e6697f2671ddbf3167fa412a03694b6d7d28267","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n44w2df8vj8t/cluster-candidate-4-21-mppmvd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c4140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mppmvd: failed waiting for cluster=\"cluster-candidate-4-21-mppmvd\" in resourcegroup=\"rg-candidate-4-21-n44w2df8vj8t\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"name\": \"5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19951964Z\",\n \"endTime\": \"2026-08-25T11:20:57.290621961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n44w2df8vj8t/cluster-candidate-4-21-mppmvd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c4140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mppmvd: failed waiting for cluster=\"cluster-candidate-4-21-mppmvd\" in resourcegroup=\"rg-candidate-4-21-n44w2df8vj8t\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"name\": \"5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.19951964Z\",\n \"endTime\": \"2026-08-25T11:20:57.290621961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"edc445226bca8c15a6801b9e4e1854b6c7f899e847f419d892bdf3d1968a9d97","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f42ee17e0fd460e6baf5060835861f5b3e5384baff9121d441a1e851dd0515e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"4bd8257d88ff066db3ad8c08e1600658f460390b8e6e515cc65d3100aa511a22","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a6c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9mf2mfq4r2d9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"name\": \"d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:12.867884764Z\",\n \"endTime\": \"2026-08-25T11:22:17.332591987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a6c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9mf2mfq4r2d9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"name\": \"d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:12.867884764Z\",\n \"endTime\": \"2026-08-25T11:22:17.332591987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d7f14514a88680ff6615f00bb81c0bec2c6dadd1b601567325ec72490caf5ca","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff1da35bf9fc1f71ed6ae041578b2d0bdbd99e83d25062505bc397d4a05f01a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","failed_at":"e2e","signature_id":"983b22f1f6c46c18080a9b4eef0fb721f28445b2ab45c04695540c6e1542253d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-twq2f7\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008b8240\u003e: \n failed to create HCP cluster cluster-zstream-4-20-twq2f7: failed waiting for cluster=\"cluster-zstream-4-20-twq2f7\" in resourcegroup=\"rg-zstream-upgrade-4-20-vnnwhcdv49hn\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7a9b211-70f5-46b5-938d-82c5e0f7d27f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7a9b211-70f5-46b5-938d-82c5e0f7d27f\",\n \"name\": \"b7a9b211-70f5-46b5-938d-82c5e0f7d27f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.235664358Z\",\n \"endTime\": \"2026-08-25T11:20:57.276791645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-twq2f7\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008b8240\u003e: \n failed to create HCP cluster cluster-zstream-4-20-twq2f7: failed waiting for cluster=\"cluster-zstream-4-20-twq2f7\" in resourcegroup=\"rg-zstream-upgrade-4-20-vnnwhcdv49hn\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7a9b211-70f5-46b5-938d-82c5e0f7d27f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7a9b211-70f5-46b5-938d-82c5e0f7d27f\",\n \"name\": \"b7a9b211-70f5-46b5-938d-82c5e0f7d27f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:01:52.235664358Z\",\n \"endTime\": \"2026-08-25T11:20:57.276791645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":30,"failure_pattern_ids":["01b6bec7cc22825d4ef87b95818e501dbd62c6d195e057bf40cb1fb4d00f77e0","02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","04ab38fb4caffee001686eb32195645f728f8555d294ffa19de0a9f505223827","1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","2256dab0b9b860af42d8774bc9e3bdf3a75814dca1080b715190d3d8ab7a2eb9","22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","5d1725423ee4b212bccb0d13858a78dd47769db5d794223ea2e26d6df7c4f5ad","66285e3570a1d3a91819718028ead75ff34527595051ad0083e72a103f62194a","691000237ed68ee2bea1eca4972f51b642a5c69ce54dd17b387ef5ef88b01453","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","7d7f14514a88680ff6615f00bb81c0bec2c6dadd1b601567325ec72490caf5ca","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","95452bdc0ded378689e5f2c12b2d45cbf916f7a4e0676d618a92d0f3d8778ac4","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","bebd56cb7971d5cbd2ba0627c63fe9925ffa3a6799fdb1d98a37f3159a2c38aa","db4713748d0354976aeef8212e3a9f4a93e3ecf500342fbbcb1fbd6a21579f7d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","edc445226bca8c15a6801b9e4e1854b6c7f899e847f419d892bdf3d1968a9d97","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":30,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192723133009920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:03:11Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e13613e167863ea810a1a9aa49cad956e0c79365969ec4f5afe2d2abd3fc4987","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192723133009920","occurred_at":"2026-08-25T10:03:11Z","failed_at":"provision","signature_id":"561505d61b3a52ebd516a8399919b0bdc92e7d97b736ef08c12b540f9f4f583c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-25T10:31:54.607Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-08-25T10:31:55.356Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-3697365572/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"bf87cfe5b41d04a13227b8cc9458e4b4b92a5088","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:03:03Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"217f906aea1bdf11b080fe1f80cb038dc1a3b0b0287d1ddda5fbee0ebe92431f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"alert","signature_id":"9a2d5923b47e722ec14a03cd82e80a5847eb6af6c56086cf441bc75f54feb7ce","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:04:49Z\n Ended: 2026-08-25T11:19:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8682368-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdibolp1edv6hvnhp9i6alddcpmqbrg-x1v3s6y8u0i9u9q\", pod=\"router-7dc45bdf49-z52rd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdibolp1edv6hvnhp9i6alddcpmqbrg-x1v3s6y8u0i9u9q/router-7dc45bdf49-z52rd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"292179c0857e8ce5aa92ff554459b5757c8d3974a70bdbf32a322fd746fcd691","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"e2e","signature_id":"59336122fa10346af3e78ba303a4d90407e92e02b272f6b3005e01628acb96b4","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8d5e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s7knrjd2n4p9\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a20d1a46-4016-4993-942c-628985f40d60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a20d1a46-4016-4993-942c-628985f40d60\",\n \"name\": \"a20d1a46-4016-4993-942c-628985f40d60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:52:09.390785108Z\",\n \"endTime\": \"2026-08-25T11:11:13.084035599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8d5e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s7knrjd2n4p9\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a20d1a46-4016-4993-942c-628985f40d60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a20d1a46-4016-4993-942c-628985f40d60\",\n \"name\": \"a20d1a46-4016-4993-942c-628985f40d60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:52:09.390785108Z\",\n \"endTime\": \"2026-08-25T11:11:13.084035599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"44bc7bee37ea4294461a929b239da42a8b8c29de252399e5f5a0c0351ca5880f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"e2e","signature_id":"eb35914289e36ada82afb95d0842ac1cbf05cda80bee90852c8426b9626b9565","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:192]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0012ca6a8\u003e: \n VerifyNodesReady failed: there are not ready nodes: [cilium-cluster-cilium-np-vknrp-c5ljh cilium-cluster-cilium-np-vknrp-thjfc]\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9jxg574dgjtb\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df94ac92-d0eb-418f-ba1a-c79aa21c8ec3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df94ac92-d0eb-418f-ba1a-c79aa21c8ec3\",\n \"name\": \"df94ac92-d0eb-418f-ba1a-c79aa21c8ec3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:59.904255754Z\",\n \"endTime\": \"2026-08-25T11:25:59.419108691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:192]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0012ca6a8\u003e: \n VerifyNodesReady failed: there are not ready nodes: [cilium-cluster-cilium-np-vknrp-c5ljh cilium-cluster-cilium-np-vknrp-thjfc]\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9jxg574dgjtb\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df94ac92-d0eb-418f-ba1a-c79aa21c8ec3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df94ac92-d0eb-418f-ba1a-c79aa21c8ec3\",\n \"name\": \"df94ac92-d0eb-418f-ba1a-c79aa21c8ec3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:59.904255754Z\",\n \"endTime\": \"2026-08-25T11:25:59.419108691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"946508202215a61660157f8022fe81844efe0daa7f532e486e7d7cc834dbee16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"e2e","signature_id":"79c7c41c97eedc396d88101d613a41fb1d71de25afa00931594fcc8ea799c229","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f88c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-kd8kq878wflk\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70626b07-0cb8-4f0d-933a-2f9445223f6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70626b07-0cb8-4f0d-933a-2f9445223f6a\",\n \"name\": \"70626b07-0cb8-4f0d-933a-2f9445223f6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:52:07.614945455Z\",\n \"endTime\": \"2026-08-25T11:11:13.090308867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f88c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-kd8kq878wflk\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70626b07-0cb8-4f0d-933a-2f9445223f6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70626b07-0cb8-4f0d-933a-2f9445223f6a\",\n \"name\": \"70626b07-0cb8-4f0d-933a-2f9445223f6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:52:07.614945455Z\",\n \"endTime\": \"2026-08-25T11:11:13.090308867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9c244584ac3e2d5aef604a7a91bf467cbb01341251895462d031ec62544ed418","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"e2e","signature_id":"938d48f5f5615798cf28a4204904814739ed9c8d4708bcdf50c0a0e5b416c33a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-jjpphbklnmpm/cluster-candidate-5-0-8jxb6p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a56220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-8jxb6p: failed waiting for cluster=\"cluster-candidate-5-0-8jxb6p\" in resourcegroup=\"rg-candidate-5-0-jjpphbklnmpm\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66297db9-5baf-4aaa-8975-b676c020ea95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66297db9-5baf-4aaa-8975-b676c020ea95\",\n \"name\": \"66297db9-5baf-4aaa-8975-b676c020ea95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:52:17.485728635Z\",\n \"endTime\": \"2026-08-25T11:11:23.092877103Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-jjpphbklnmpm/cluster-candidate-5-0-8jxb6p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a56220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-8jxb6p: failed waiting for cluster=\"cluster-candidate-5-0-8jxb6p\" in resourcegroup=\"rg-candidate-5-0-jjpphbklnmpm\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66297db9-5baf-4aaa-8975-b676c020ea95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66297db9-5baf-4aaa-8975-b676c020ea95\",\n \"name\": \"66297db9-5baf-4aaa-8975-b676c020ea95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:52:17.485728635Z\",\n \"endTime\": \"2026-08-25T11:11:23.092877103Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cadc67d876b5828f9558b0355f42f22405d7a872372d986c61212f9de411b6a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"e2e","signature_id":"4806e756ed0578671bc1be64618a268b2aa1f613e22beef835379c8a977148f3","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-5cwprg\" for cluster \"cluster-hs-pre-5cwprg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010169e0\u003e: \n failed to create NodePool np-5cwprg: failed waiting for nodepool=\"np-5cwprg\" for cluster \"cluster-hs-pre-5cwprg\" in resourcegroup=\"rg-hs-pre--lqnv4n95tx47\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40364dda-e147-4d07-941d-26fe941e32ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40364dda-e147-4d07-941d-26fe941e32ab\",\n \"name\": \"40364dda-e147-4d07-941d-26fe941e32ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:14.906937807Z\",\n \"endTime\": \"2026-08-25T11:26:19.497689075Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-5cwprg\" for cluster \"cluster-hs-pre-5cwprg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010169e0\u003e: \n failed to create NodePool np-5cwprg: failed waiting for nodepool=\"np-5cwprg\" for cluster \"cluster-hs-pre-5cwprg\" in resourcegroup=\"rg-hs-pre--lqnv4n95tx47\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40364dda-e147-4d07-941d-26fe941e32ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40364dda-e147-4d07-941d-26fe941e32ab\",\n \"name\": \"40364dda-e147-4d07-941d-26fe941e32ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:14.906937807Z\",\n \"endTime\": \"2026-08-25T11:26:19.497689075Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f2437ce5d85aa1524eb0acf7fe9345866cb396e477e15d9a623eda8a9c5d922c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","failed_at":"e2e","signature_id":"da59667e1510570b8a7920b7fcdf7406c1b2daaf77247d80291d5db3549304d3","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015281e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fmb6zrrk7zsv\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f451991-d984-48c4-98fe-f9cd0fb13d1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f451991-d984-48c4-98fe-f9cd0fb13d1a\",\n \"name\": \"3f451991-d984-48c4-98fe-f9cd0fb13d1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:18.751416822Z\",\n \"endTime\": \"2026-08-25T11:26:19.532469118Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015281e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fmb6zrrk7zsv\" to finish creating: GET https://rp.j8682368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f451991-d984-48c4-98fe-f9cd0fb13d1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f451991-d984-48c4-98fe-f9cd0fb13d1a\",\n \"name\": \"3f451991-d984-48c4-98fe-f9cd0fb13d1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:18.751416822Z\",\n \"endTime\": \"2026-08-25T11:26:19.532469118Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:03:00Z"},"failed_at":["alert","e2e"],"failed_test_count":52,"occurrences":[{"row_id":"02c69cd0580e69c9dc5ac40722a9cad3d8184a6eedb1bef39dde78c9d6f8943e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"26682ebc65e6771f4deee3248fa461bb0de542a4ab0eebfc70762ad29bf4f7f8","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-g2j9ht at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c2e9e0\u003e: \n failed to create HCP cluster np-skip-minor-g2j9ht: failed waiting for cluster=\"np-skip-minor-g2j9ht\" in resourcegroup=\"rg-np-skip-minor-g2j9ht-c89fhbmn6zb7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/104bf35c-3c35-49bd-a295-bd89c97479d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/104bf35c-3c35-49bd-a295-bd89c97479d4\",\n \"name\": \"104bf35c-3c35-49bd-a295-bd89c97479d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:08.489918597Z\",\n \"endTime\": \"2026-08-25T11:24:17.594649285Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-storage-operator, packageserver, cluster-network-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, catalog-operator, olm-operator, community-operators-catalog, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-g2j9ht at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c2e9e0\u003e: \n failed to create HCP cluster np-skip-minor-g2j9ht: failed waiting for cluster=\"np-skip-minor-g2j9ht\" in resourcegroup=\"rg-np-skip-minor-g2j9ht-c89fhbmn6zb7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/104bf35c-3c35-49bd-a295-bd89c97479d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/104bf35c-3c35-49bd-a295-bd89c97479d4\",\n \"name\": \"104bf35c-3c35-49bd-a295-bd89c97479d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:08.489918597Z\",\n \"endTime\": \"2026-08-25T11:24:17.594649285Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-storage-operator, packageserver, cluster-network-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, catalog-operator, olm-operator, community-operators-catalog, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"02f6b9eea02a1ea8215057bbd7c47823e9321fbfa551c945216d47c7a89915f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"7311a955c2510af435a3dd4e8bda65a101a0cfae9846f5be659992b940243062","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-kp6cj8gslgn7/cluster-candidate-4-20-2ftmsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101ebe0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-2ftmsx: failed waiting for cluster=\"cluster-candidate-4-20-2ftmsx\" in resourcegroup=\"rg-candidate-4-20-kp6cj8gslgn7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"name\": \"9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.525443298Z\",\n \"endTime\": \"2026-08-25T11:24:07.636192222Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-kp6cj8gslgn7/cluster-candidate-4-20-2ftmsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101ebe0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-2ftmsx: failed waiting for cluster=\"cluster-candidate-4-20-2ftmsx\" in resourcegroup=\"rg-candidate-4-20-kp6cj8gslgn7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"name\": \"9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.525443298Z\",\n \"endTime\": \"2026-08-25T11:24:07.636192222Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55a75f995426be3c05299ad4947251679d4a99ba7e77d726aeadaba9bd31ae5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2010eb5acd34dc849eb9d03b22c1d4faff82b64fe07bfd27e9d3a25f590a1b30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"28b9c82589a4dda1452e567d40ab21019c6cee221e37e01472de1dab7d5143df","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-sjzzj9kjw9kf/cluster-candidate-5-0-rjlgtk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a660\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rjlgtk: failed waiting for cluster=\"cluster-candidate-5-0-rjlgtk\" in resourcegroup=\"rg-candidate-5-0-sjzzj9kjw9kf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\",\n \"name\": \"8dd830d6-347e-4473-bb10-124104e89a73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.05360308Z\",\n \"endTime\": \"2026-08-25T11:23:57.532994811Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-sjzzj9kjw9kf/cluster-candidate-5-0-rjlgtk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a660\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rjlgtk: failed waiting for cluster=\"cluster-candidate-5-0-rjlgtk\" in resourcegroup=\"rg-candidate-5-0-sjzzj9kjw9kf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\",\n \"name\": \"8dd830d6-347e-4473-bb10-124104e89a73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.05360308Z\",\n \"endTime\": \"2026-08-25T11:23:57.532994811Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b393a3822cea98a3f0355403abde765ef631da69f047c1828358b6194172cb6b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"25ceeb0f68b119560ef2b1d004e0506cd0b5f380d9ba0413936e4cb718603da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"18f9ebc5eb73ff84a02d71ec227c141bf94f7ba02eeea0d33eed11a56429d19b","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010003c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dgghk7xd8fkc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"name\": \"813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.708804056Z\",\n \"endTime\": \"2026-08-25T11:24:07.512424839Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010003c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dgghk7xd8fkc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"name\": \"813de9e6-c926-4fa8-a38a-500226e3059a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.708804056Z\",\n \"endTime\": \"2026-08-25T11:24:07.512424839Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8fcfad7ee636c64bcc7ec16c2a0da480eb90b3d648cf1239ba480534695bb74a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b0cb600ec9b17f70aa04948ad6deffc0d1a27d2ff9395a0eec35c4a328a0610","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"504ae63c8f089cb82a68d41adbc6f1b843f3855c8b1b37f224907c01e60811f8","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-4gfnzd with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b190c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-4gfnzd: failed waiting for cluster=\"np-version-upgrade-cluster-4gfnzd\" in resourcegroup=\"rg-np-version-upgrade-4gfnzd-rj8pxrlvvg4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d916a34c-5fd9-4150-b24c-9e76f99b04aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d916a34c-5fd9-4150-b24c-9e76f99b04aa\",\n \"name\": \"d916a34c-5fd9-4150-b24c-9e76f99b04aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.664010297Z\",\n \"endTime\": \"2026-08-25T11:23:57.583359659Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, olm-operator, packageserver, dns-operator, catalog-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-4gfnzd with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b190c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-4gfnzd: failed waiting for cluster=\"np-version-upgrade-cluster-4gfnzd\" in resourcegroup=\"rg-np-version-upgrade-4gfnzd-rj8pxrlvvg4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d916a34c-5fd9-4150-b24c-9e76f99b04aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d916a34c-5fd9-4150-b24c-9e76f99b04aa\",\n \"name\": \"d916a34c-5fd9-4150-b24c-9e76f99b04aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.664010297Z\",\n \"endTime\": \"2026-08-25T11:23:57.583359659Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, olm-operator, packageserver, dns-operator, catalog-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"338316a5db8f9a0cfa98dcf7538e259ba30a4c62b4e83c920fb5c89dbc788c19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"15cec8044c988cc97a4d3ec832848e5522b08071d86565ff2ec35915e8e8e5ea","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cd4b20\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nfq89d68b2j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"name\": \"6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.362546319Z\",\n \"endTime\": \"2026-08-25T11:24:07.656848362Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, olm-operator, cluster-storage-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, packageserver, dns-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cd4b20\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nfq89d68b2j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"name\": \"6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.362546319Z\",\n \"endTime\": \"2026-08-25T11:24:07.656848362Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, olm-operator, cluster-storage-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, packageserver, dns-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"34fc71c6a788270bb192ea9c07684fe41102d1d3a6d0c1a7a96aff4329e3a25d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"4bc6fa9b62fea5fb0a886bde62212833e54bd33c1122e0a8839c522e31ede058","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000976d60\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sp25f888spq4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"name\": \"d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.802251942Z\",\n \"endTime\": \"2026-08-25T11:24:07.715404645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000976d60\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sp25f888spq4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"name\": \"d3198a15-7882-4985-8fb9-601f276da5ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.802251942Z\",\n \"endTime\": \"2026-08-25T11:24:07.715404645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f59d41db8c7110a2a66df7a04a929104d4f74b17a121064979556d85ac966c14","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"386ef2b8fa22ed054e7f167dd8fa5709909bf14fd738dbcd835a61ca87ccb809","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"f7a0795ccf41d2868c0062e96d14bdc47a40e0b4d69128fb529e4080a9cfca2a","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b26780\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-lzwn2n84w4rd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49b0ca7-b350-4e35-b5c4-7a6cc066dabb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49b0ca7-b350-4e35-b5c4-7a6cc066dabb\",\n \"name\": \"a49b0ca7-b350-4e35-b5c4-7a6cc066dabb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.912879528Z\",\n \"endTime\": \"2026-08-25T11:24:07.72071407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b26780\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-lzwn2n84w4rd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49b0ca7-b350-4e35-b5c4-7a6cc066dabb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49b0ca7-b350-4e35-b5c4-7a6cc066dabb\",\n \"name\": \"a49b0ca7-b350-4e35-b5c4-7a6cc066dabb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.912879528Z\",\n \"endTime\": \"2026-08-25T11:24:07.72071407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3946c3112b255b886c88c0a6d9638f1679f2cabf8dfc4f684ae1f26a2ce73399","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"8cb8819d08e22597597cadf9ee8ec483916ec2e91ed7308d6b08f79ac3c67f38","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fceaa0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4scl6wzd5ltc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7f2e105-1507-4d15-a29b-003e59886d0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7f2e105-1507-4d15-a29b-003e59886d0b\",\n \"name\": \"c7f2e105-1507-4d15-a29b-003e59886d0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.33753064Z\",\n \"endTime\": \"2026-08-25T11:23:47.467774327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fceaa0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4scl6wzd5ltc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7f2e105-1507-4d15-a29b-003e59886d0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7f2e105-1507-4d15-a29b-003e59886d0b\",\n \"name\": \"c7f2e105-1507-4d15-a29b-003e59886d0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.33753064Z\",\n \"endTime\": \"2026-08-25T11:23:47.467774327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b1cb73bda54ff28e4a6ca00e97e049e23c058e1efff778844861e8cc8cf9d40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"e7d58e0615757ed6b9924d8c23510714583aa7bdf89e2194c35463a6520b17a5","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d08560\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mx5c2xs4fbj7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"name\": \"0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:26.179895607Z\",\n \"endTime\": \"2026-08-25T11:25:27.51902962Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d08560\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mx5c2xs4fbj7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"name\": \"0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:26.179895607Z\",\n \"endTime\": \"2026-08-25T11:25:27.51902962Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a4546d1ccb11eb2bd205f13e52f18ff56616b802ad979a8ada123b2e8ac5a07","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"43dfc0be12d2c217f5d99f8546f47df447c9bf377ca928cb381b8f5fb8edb0bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"93f75a595fe834ff90d354323b1795c64c6aa1fffff5420185d0cc08b6045965","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-x7w9rq with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000218ae0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-x7w9rq: failed waiting for cluster=\"np-version-upgrade-cluster-x7w9rq\" in resourcegroup=\"rg-np-version-upgrade-x7w9rq-dvf8r82dxzzq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"name\": \"96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:57.577943839Z\",\n \"endTime\": \"2026-08-25T11:23:57.610020208Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: certified-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-x7w9rq with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000218ae0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-x7w9rq: failed waiting for cluster=\"np-version-upgrade-cluster-x7w9rq\" in resourcegroup=\"rg-np-version-upgrade-x7w9rq-dvf8r82dxzzq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"name\": \"96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:57.577943839Z\",\n \"endTime\": \"2026-08-25T11:23:57.610020208Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: certified-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56ac1ffd8f7c84d6800647347b139a137e691a55ff2e3d1faa305a6b07101364","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4521d7323ba87cd3532ec3bcf544dbf54a5fff46a1eacde0a2f58b9dd9c0e2b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"a34611ae2bd1afe363e9b85e5d346d6e037222fe5510e7c418c4c0d0a2de08f2","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84820\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b9lzcpbpn5wp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"name\": \"2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.392903067Z\",\n \"endTime\": \"2026-08-25T11:24:07.668839785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84820\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b9lzcpbpn5wp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"name\": \"2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.392903067Z\",\n \"endTime\": \"2026-08-25T11:24:07.668839785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"92ae44f2e86af5544dec422ded7c331d2f560f22fa4954538768a565c71a9dbd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5370877610e8ad60c977f87d6b87a0c20a253fb06266cc90b4fdf9810ea5926c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"47245e609d7874f7d25c869a473a1e7e721abc0d258fe07dc19210de937d516d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-t52ww6\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00183c860\u003e: \n failed to create HCP cluster cluster-zstream-4-22-t52ww6: failed waiting for cluster=\"cluster-zstream-4-22-t52ww6\" in resourcegroup=\"rg-zstream-upgrade-4-22-rdnc78mtxx5j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"name\": \"9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.248454811Z\",\n \"endTime\": \"2026-08-25T11:24:07.701920992Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: cluster-image-registry-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-t52ww6\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00183c860\u003e: \n failed to create HCP cluster cluster-zstream-4-22-t52ww6: failed waiting for cluster=\"cluster-zstream-4-22-t52ww6\" in resourcegroup=\"rg-zstream-upgrade-4-22-rdnc78mtxx5j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"name\": \"9c696222-49a6-4ca9-b706-8e5a57435480\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.248454811Z\",\n \"endTime\": \"2026-08-25T11:24:07.701920992Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: cluster-image-registry-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c4fdcee2df11b558d3b3164768a55e10395e8648b1940e20c553175bae5d91ab","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"55693de04695aeb74a07b771d80210213e8e3c5bf08ff1039e11923193d9562d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"6a014dc04f005c0ce2a8189b9d8c8434c9d29f69cb03bf76fae5f1ea1b289999","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-rm8r69s8dhfz/cluster-candidate-4-21-q2f2xq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000902920\u003e: \n failed to create HCP cluster cluster-candidate-4-21-q2f2xq: failed waiting for cluster=\"cluster-candidate-4-21-q2f2xq\" in resourcegroup=\"rg-candidate-4-21-rm8r69s8dhfz\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c21efcc-c66c-4246-a3a7-827947fedab9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c21efcc-c66c-4246-a3a7-827947fedab9\",\n \"name\": \"5c21efcc-c66c-4246-a3a7-827947fedab9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.044151568Z\",\n \"endTime\": \"2026-08-25T11:24:07.669127251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-rm8r69s8dhfz/cluster-candidate-4-21-q2f2xq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000902920\u003e: \n failed to create HCP cluster cluster-candidate-4-21-q2f2xq: failed waiting for cluster=\"cluster-candidate-4-21-q2f2xq\" in resourcegroup=\"rg-candidate-4-21-rm8r69s8dhfz\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c21efcc-c66c-4246-a3a7-827947fedab9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c21efcc-c66c-4246-a3a7-827947fedab9\",\n \"name\": \"5c21efcc-c66c-4246-a3a7-827947fedab9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.044151568Z\",\n \"endTime\": \"2026-08-25T11:24:07.669127251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"57eeaf448e1c3bfbd7a8b809e5dc0ca3e94f70892e700c6bee187dad026ab563","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"alert","signature_id":"77f2e0ebb4f832bfccfdc5298c2a752e61abc579b540c5ba9eb248bda7ab6b07","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:55:05Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j2121984-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2121984/providers/microsoft.monitor/accounts/services-j2121984\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"601fc1af6b8afe4853ee141e9b3993b8d21a6a2b1e289ca33c6717ed3e74acdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"1f069fa0311578db887619d6e5fa3d28f357906bf03f02fdb94770dedff8043b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pgpvxwpm6mjj/cluster-candidate-4-22-bnptgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084220\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bnptgd: failed waiting for cluster=\"cluster-candidate-4-22-bnptgd\" in resourcegroup=\"rg-candidate-4-22-pgpvxwpm6mjj\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"name\": \"9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.515128304Z\",\n \"endTime\": \"2026-08-25T11:24:07.660901963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, openshift-controller-manager, cluster-storage-operator, olm-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pgpvxwpm6mjj/cluster-candidate-4-22-bnptgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084220\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bnptgd: failed waiting for cluster=\"cluster-candidate-4-22-bnptgd\" in resourcegroup=\"rg-candidate-4-22-pgpvxwpm6mjj\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"name\": \"9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.515128304Z\",\n \"endTime\": \"2026-08-25T11:24:07.660901963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, openshift-controller-manager, cluster-storage-operator, olm-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a476f5844d3ff4814223898b88783de6ad4d76543b3ffd3ee48eaebeacb8ae1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"e70a8923551947fd69aed4662e45c5ae1a351669ab5e6230b53b2fb9b06f69bf","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3ed80\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-klhpfqtd5gz6\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f19d77e4-27f1-47c7-ae51-adcc966d0da7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f19d77e4-27f1-47c7-ae51-adcc966d0da7\",\n \"name\": \"f19d77e4-27f1-47c7-ae51-adcc966d0da7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.024004396Z\",\n \"endTime\": \"2026-08-25T11:23:57.629775108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3ed80\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-klhpfqtd5gz6\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f19d77e4-27f1-47c7-ae51-adcc966d0da7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f19d77e4-27f1-47c7-ae51-adcc966d0da7\",\n \"name\": \"f19d77e4-27f1-47c7-ae51-adcc966d0da7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.024004396Z\",\n \"endTime\": \"2026-08-25T11:23:57.629775108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6e529814c45d716e7fa966e7ee0095756d690e2263eab227b8553eb2cc357bbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"301d0f6d3f61c8fc4bfdf4b36d3da0f6a5670b9ba6b385d3c18bb6644f1afcaf","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011b8160\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5zct5rkc97n4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00460c52-4267-4ccc-90ce-ef17247c788e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00460c52-4267-4ccc-90ce-ef17247c788e\",\n \"name\": \"00460c52-4267-4ccc-90ce-ef17247c788e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.594782601Z\",\n \"endTime\": \"2026-08-25T11:23:57.646969908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011b8160\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5zct5rkc97n4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00460c52-4267-4ccc-90ce-ef17247c788e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00460c52-4267-4ccc-90ce-ef17247c788e\",\n \"name\": \"00460c52-4267-4ccc-90ce-ef17247c788e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.594782601Z\",\n \"endTime\": \"2026-08-25T11:23:57.646969908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6e8b317e3aa391627c5f6b652fceee4dbaf741fc15a0633f3192ddd3efdffad7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"841f5d6b1662eac99a6cc527de5c5a356bc8edcef21537485d177eab3f47e0b8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7cac0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-t54b5mhfsfw7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/585a4096-9e4c-4106-b5b2-f0f9922f81a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/585a4096-9e4c-4106-b5b2-f0f9922f81a9\",\n \"name\": \"585a4096-9e4c-4106-b5b2-f0f9922f81a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.951801482Z\",\n \"endTime\": \"2026-08-25T11:23:47.66244911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7cac0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-t54b5mhfsfw7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/585a4096-9e4c-4106-b5b2-f0f9922f81a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/585a4096-9e4c-4106-b5b2-f0f9922f81a9\",\n \"name\": \"585a4096-9e4c-4106-b5b2-f0f9922f81a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.951801482Z\",\n \"endTime\": \"2026-08-25T11:23:47.66244911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"714840e72161d0f87d35c92785ce10b12de7e7cc41c366d0ddb296a7aa39d376","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"aabd7b54cdc1ced5ff8794a6e08fcc1a9d50f3d188a6e28369345aad73a983a2","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001908a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dcpt8jbnmqxn\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07a64aff-4896-425c-bd97-6b0eb5c713b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07a64aff-4896-425c-bd97-6b0eb5c713b8\",\n \"name\": \"07a64aff-4896-425c-bd97-6b0eb5c713b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:27.482080484Z\",\n \"endTime\": \"2026-08-25T11:25:27.710374186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001908a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dcpt8jbnmqxn\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07a64aff-4896-425c-bd97-6b0eb5c713b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07a64aff-4896-425c-bd97-6b0eb5c713b8\",\n \"name\": \"07a64aff-4896-425c-bd97-6b0eb5c713b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:27.482080484Z\",\n \"endTime\": \"2026-08-25T11:25:27.710374186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75e7db645d82fc5aedc2bbabafc79b23a10202c3936636c3c7c01184e0759413","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"20fc32ee45196a9ed01e8c936d2dde739ba00031d099ddd39be043bfecacc1ed","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006eefa0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-blffsk8cqz4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"name\": \"7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.01277565Z\",\n \"endTime\": \"2026-08-25T11:24:07.575415707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006eefa0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-blffsk8cqz4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"name\": \"7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.01277565Z\",\n \"endTime\": \"2026-08-25T11:24:07.575415707Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7be6b09c6bbb28adba99c6de7dfc1604dfce000bc6664dedbae8e8b9b0253fa6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7c1829d736bad4e2997613e71667835e0af22870232518130f3691369c38e9e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"674600a738c7667770e739f9055e1d79ea50c97b5b16436d10f3b9df4e0297d0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h8x2lnscjddq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"name\": \"306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:01.374469415Z\",\n \"endTime\": \"2026-08-25T11:24:07.731322209Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56680\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h8x2lnscjddq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"name\": \"306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:01.374469415Z\",\n \"endTime\": \"2026-08-25T11:24:07.731322209Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75a2ddc5380cf22cdc0f7eee734c0875c6030bb3df880d9c39c49bb360b07d00","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, ignition-server-proxy, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e4f1c77d0c2ba014e8cd2b53e8c777a162a3a8da8c210ef93d0d5fde5d8a144","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"f1d2bc77809969c73b592e28941870857aa223052d643ab35baa352ec051a614","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c189e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r5zg79zklshc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"name\": \"460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.369521114Z\",\n \"endTime\": \"2026-08-25T11:24:07.709199647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001c189e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r5zg79zklshc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"name\": \"460dce50-0e02-4f7d-918e-303c53d250cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.369521114Z\",\n \"endTime\": \"2026-08-25T11:24:07.709199647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84c228108e2a74c3eb7ecf22976d7f04c8d3e1f462e4eb41f272c0aff211300b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"888bc67776cbd994161885ff23ba3a296fc9a21c443334126b85b45081b8aa68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"fe41252ce6ed75f9abb776a92d3943f65f76fc05052906f4fda9f43fe1b11ce7","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012005e0\u003e: \n failed to create HCP cluster np-downgrade-sxtp4r: failed waiting for cluster=\"np-downgrade-sxtp4r\" in resourcegroup=\"rg-np-downgrade-sxtp4r-fhklq2vsgmt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"name\": \"34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:47.779072949Z\",\n \"endTime\": \"2026-08-25T11:23:47.559079763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012005e0\u003e: \n failed to create HCP cluster np-downgrade-sxtp4r: failed waiting for cluster=\"np-downgrade-sxtp4r\" in resourcegroup=\"rg-np-downgrade-sxtp4r-fhklq2vsgmt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"name\": \"34134851-0102-4d35-97b3-944ff4b7f92e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:47.779072949Z\",\n \"endTime\": \"2026-08-25T11:23:47.559079763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a284dafa6b52e08618be50123ccc8a887a57e4ea0bfe5fc4413e3713853ec5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"a6d52fac7766cec3d727e2f0d94855f4d0b82cda86272b0009927fee77388e73","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f38340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4zrvgxxs9sbf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"name\": \"b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:16.036728583Z\",\n \"endTime\": \"2026-08-25T11:24:17.88745643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-route-controller-manager, redhat-operators-catalog, olm-operator, community-operators-catalog, cluster-policy-controller, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-storage-operator, packageserver, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f38340\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4zrvgxxs9sbf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"name\": \"b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:16.036728583Z\",\n \"endTime\": \"2026-08-25T11:24:17.88745643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-route-controller-manager, redhat-operators-catalog, olm-operator, community-operators-catalog, cluster-policy-controller, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-storage-operator, packageserver, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d86986e713d0617aead4d49d221d7d7ca33ab0b85235c167d3cbe122fa03f9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"2883b2d7f8a4d0400992958856aeb456b0ab19cc4ff95a4341bb724e485a3dc6","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001100500\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-29j67455cd7q\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"name\": \"f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.29234345Z\",\n \"endTime\": \"2026-08-25T11:24:07.6554084Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001100500\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-29j67455cd7q\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"name\": \"f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.29234345Z\",\n \"endTime\": \"2026-08-25T11:24:07.6554084Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c975965241af31c57ec44c3fff1102f75c167bb738954ccc985f5a6bb7a81cd2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fe694f425fc0f3ce7fa5538dcc4d00e154f1ca95994883aa653717042102e20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"036e61e55afe3d161610066648e9dacc1c80f9cbd63ca3be6cf42026860462b7","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-cj86x9\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a38640\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-cj86x9: failed waiting for cluster=\"cp-ystream-upgrade-4-21-cj86x9\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-4kqr78rwwtdk\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df914af8-60c2-4694-8fd1-ddb67d75fb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df914af8-60c2-4694-8fd1-ddb67d75fb05\",\n \"name\": \"df914af8-60c2-4694-8fd1-ddb67d75fb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:36.064777855Z\",\n \"endTime\": \"2026-08-25T11:23:37.679439509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-cj86x9\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a38640\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-cj86x9: failed waiting for cluster=\"cp-ystream-upgrade-4-21-cj86x9\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-4kqr78rwwtdk\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df914af8-60c2-4694-8fd1-ddb67d75fb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df914af8-60c2-4694-8fd1-ddb67d75fb05\",\n \"name\": \"df914af8-60c2-4694-8fd1-ddb67d75fb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:36.064777855Z\",\n \"endTime\": \"2026-08-25T11:23:37.679439509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"95eedccf58f32929b22d63819ce228a7b3ebc1631bdd7e9a25847eaff31f486e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"79463fb4058b95541bcd55a0c46b6475494bc057241f46f47f685884fbe58103","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jp97dz\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105a4a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-jp97dz: failed waiting for cluster=\"cluster-zstream-4-21-jp97dz\" in resourcegroup=\"rg-zstream-upgrade-4-21-9x6gv5vxjxw2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8798bc57-9343-430d-a747-ca9fa46740a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8798bc57-9343-430d-a747-ca9fa46740a4\",\n \"name\": \"8798bc57-9343-430d-a747-ca9fa46740a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:00.721933214Z\",\n \"endTime\": \"2026-08-25T11:24:07.72807329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jp97dz\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105a4a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-jp97dz: failed waiting for cluster=\"cluster-zstream-4-21-jp97dz\" in resourcegroup=\"rg-zstream-upgrade-4-21-9x6gv5vxjxw2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8798bc57-9343-430d-a747-ca9fa46740a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8798bc57-9343-430d-a747-ca9fa46740a4\",\n \"name\": \"8798bc57-9343-430d-a747-ca9fa46740a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:00.721933214Z\",\n \"endTime\": \"2026-08-25T11:24:07.72807329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9885420b890bccd6746a8fc17f163979c1b2b9e2e14fb480243633c21f335dd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"f935555cb57f59b6764c10d1c7fa203076af04b63661aa2b31d669f4c7cdb56f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wndrtf\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00132c740\u003e: \n failed to create HCP cluster cluster-zstream-4-20-wndrtf: failed waiting for cluster=\"cluster-zstream-4-20-wndrtf\" in resourcegroup=\"rg-zstream-upgrade-4-20-btw2nf2rz2z4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"name\": \"a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.275646571Z\",\n \"endTime\": \"2026-08-25T11:24:07.685798394Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wndrtf\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00132c740\u003e: \n failed to create HCP cluster cluster-zstream-4-20-wndrtf: failed waiting for cluster=\"cluster-zstream-4-20-wndrtf\" in resourcegroup=\"rg-zstream-upgrade-4-20-btw2nf2rz2z4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"name\": \"a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:06.275646571Z\",\n \"endTime\": \"2026-08-25T11:24:07.685798394Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b393a3822cea98a3f0355403abde765ef631da69f047c1828358b6194172cb6b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d28d08ead6def751e2b95290b2483bd84410b99cba1d67b35192858fbc1617c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"76e7fe8a79658b5944b75f155ea6ce85c0f606666b8630971299524820b46509","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81e60\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sx5z6bzj9gt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"name\": \"9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:17.129504958Z\",\n \"endTime\": \"2026-08-25T11:24:17.573823206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81e60\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sx5z6bzj9gt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"name\": \"9107795d-3348-494d-85dd-e9b48aaf131e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:17.129504958Z\",\n \"endTime\": \"2026-08-25T11:24:17.573823206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8172b3b938ddc92e532decd443ed82e33579c57cbdab74ca3567337de4aab992","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f29cd112d5c9dd8b096ea38dda8602a40c9813de7e09a5c113dee416a544472","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"3286e3629eaf6809d169cb339959b7df55417659272206300cd3912a2d082c76","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a2ee20\u003e: \n failed to create HCP cluster np-dg-minor-8wczf9: failed waiting for cluster=\"np-dg-minor-8wczf9\" in resourcegroup=\"rg-np-dg-minor-8wczf9-z98rv7l55r2s\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"name\": \"35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.632569496Z\",\n \"endTime\": \"2026-08-25T11:24:07.68601483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a2ee20\u003e: \n failed to create HCP cluster np-dg-minor-8wczf9: failed waiting for cluster=\"np-dg-minor-8wczf9\" in resourcegroup=\"rg-np-dg-minor-8wczf9-z98rv7l55r2s\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"name\": \"35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:58.632569496Z\",\n \"endTime\": \"2026-08-25T11:24:07.68601483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, cluster-image-registry-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b3d328e9a97f3f976227e1f892def10b7f6c53bda848b798f9760113f189d2a4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a0b74891f2654f34af3298542d86c5474b26a6f8d540f9c57c844dcf42ac14d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"2a0367cc0e9683bd02dfae7b534805fb13ea99e1bfc239a750d74f7845bd9014","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00168cbc0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w6bmf26mtnwq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e950dfd5-2c0f-4378-b9fd-609f947fd461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e950dfd5-2c0f-4378-b9fd-609f947fd461\",\n \"name\": \"e950dfd5-2c0f-4378-b9fd-609f947fd461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:47.491403607Z\",\n \"endTime\": \"2026-08-25T11:23:47.628860736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00168cbc0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w6bmf26mtnwq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e950dfd5-2c0f-4378-b9fd-609f947fd461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e950dfd5-2c0f-4378-b9fd-609f947fd461\",\n \"name\": \"e950dfd5-2c0f-4378-b9fd-609f947fd461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:47.491403607Z\",\n \"endTime\": \"2026-08-25T11:23:47.628860736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a25b48547fade2555430cbf762c9546c3c691281e98863210d3af92aac44927d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"6f6befa657f715c3b4b4b66b7a559071e94c36c338612799fd64e9fc60a3ceb0","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-qpdbsl with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e015c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qpdbsl: failed waiting for cluster=\"np-version-upgrade-cluster-qpdbsl\" in resourcegroup=\"rg-np-version-upgrade-qpdbsl-xxx55fsnfv5d\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"name\": \"35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.081081979Z\",\n \"endTime\": \"2026-08-25T11:24:07.516329781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-qpdbsl with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e015c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qpdbsl: failed waiting for cluster=\"np-version-upgrade-cluster-qpdbsl\" in resourcegroup=\"rg-np-version-upgrade-qpdbsl-xxx55fsnfv5d\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"name\": \"35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.081081979Z\",\n \"endTime\": \"2026-08-25T11:24:07.516329781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"183ac5e7e1c7f43ad252fd54e6e3fae5ee8421fc47486bdad5d091cff0dbf9a9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a721aec76a059d8665216ef1c88bb0162baa81caced5608ee27c8df79c8842bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"d6a56eb98a1093f3a5c49d3962df7ba45a52a81222c930c3ba06aff802dd1a1f","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00058a120\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hbkfwxhrsvb2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"name\": \"51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:15.619518812Z\",\n \"endTime\": \"2026-08-25T11:24:17.933561304Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00058a120\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hbkfwxhrsvb2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"name\": \"51ad145b-c45f-488c-9b63-9a804b8af221\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:15.619518812Z\",\n \"endTime\": \"2026-08-25T11:24:17.933561304Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9917e0e4cec099cb18cbbcab5a48ffe13a11b43e10f38323888849ed9a27dd49","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, openshift-route-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9e01afdcbab2fb2761b1dffec485b67d36c95f7354979bcb45a63e677b2e4be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"32b3f6d0980058b52ca8a27936b8fa3d683b85efd75564d377af68c930a114db","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001580b40\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-t82c95mkn57k\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36a63320-22f0-4cb1-8ba5-4acf72f853e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36a63320-22f0-4cb1-8ba5-4acf72f853e4\",\n \"name\": \"36a63320-22f0-4cb1-8ba5-4acf72f853e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.165986465Z\",\n \"endTime\": \"2026-08-25T11:23:57.660959397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, catalog-operator, packageserver, olm-operator, cluster-version-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001580b40\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-t82c95mkn57k\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36a63320-22f0-4cb1-8ba5-4acf72f853e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36a63320-22f0-4cb1-8ba5-4acf72f853e4\",\n \"name\": \"36a63320-22f0-4cb1-8ba5-4acf72f853e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:56.165986465Z\",\n \"endTime\": \"2026-08-25T11:23:57.660959397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, catalog-operator, packageserver, olm-operator, cluster-version-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b2fc638e1eb482f88491c311ea2aa08cc5cbb162d97f7abfd5a67d408a71f5ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"alert","signature_id":"8726d474691b88bafaa41f02f06ab47cee7673893f1cb6c5f94125da19bae517","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:31:06Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors1h5m\", cluster=\"ci01-j2121984-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2121984/providers/microsoft.monitor/accounts/services-j2121984\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of cluster create (install) operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours. A regional install failure of this magnitude typically points at a shared dependency (e.g. registry, DNS, or ARM) rather than individual clusters.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470aab7d7a3605e8633a637a70d37da16684c6630946576af6d433f851290640","failure_pattern":"alert [svc] UJClusterProvisionErrors1h5m fired","search_query":"alert [svc] UJClusterProvisionErrors1h5m fired"}},{"row_id":"b73d666eb72bcf92d97455eb1489ef81eab1b8e9b83814e5d1c57e4e583736ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"41cf799b841c41ca021361bf55e60796cb2a72779166c6626d1c1aedefd1fe20","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00059c3e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-24c5bjzcljlb\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"name\": \"b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.381643689Z\",\n \"endTime\": \"2026-08-25T11:23:57.677492232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00059c3e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-24c5bjzcljlb\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"name\": \"b21aa181-e22f-4f73-9442-34e06ecf2205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.381643689Z\",\n \"endTime\": \"2026-08-25T11:23:57.677492232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"279b8dc059ffca13ba2e7632bee0bb38b6b6e79fa3da4d820708453f05c04b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb4a052917fff1cfac422b9b0a9ab01473d72de9a93ec249fa4d34765e8dc30e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"ddfd07ad7c5094cf706dd735ad3c3aed8ba35bd4d8f19d89a430d7152ef2a389","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9khvwrw7r7ct/cluster-hs-pre-55dn5b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001046980\u003e: \n failed to create HCP cluster cluster-hs-pre-55dn5b: failed waiting for cluster=\"cluster-hs-pre-55dn5b\" in resourcegroup=\"rg-hs-pre--9khvwrw7r7ct\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b00a246e-816f-415f-ad18-11833d6b8316\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b00a246e-816f-415f-ad18-11833d6b8316\",\n \"name\": \"b00a246e-816f-415f-ad18-11833d6b8316\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.925330051Z\",\n \"endTime\": \"2026-08-25T11:24:07.740218263Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9khvwrw7r7ct/cluster-hs-pre-55dn5b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001046980\u003e: \n failed to create HCP cluster cluster-hs-pre-55dn5b: failed waiting for cluster=\"cluster-hs-pre-55dn5b\" in resourcegroup=\"rg-hs-pre--9khvwrw7r7ct\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b00a246e-816f-415f-ad18-11833d6b8316\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b00a246e-816f-415f-ad18-11833d6b8316\",\n \"name\": \"b00a246e-816f-415f-ad18-11833d6b8316\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.925330051Z\",\n \"endTime\": \"2026-08-25T11:24:07.740218263Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c61d5ac4538f7ba44f24212d46ae8711fb604c0b7e259075bf98733de3f57ba3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"812cfe1df0bd2944c0ae39c8f4dc119f0c86e898e12e7120fca50c7be2fd1a6a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-wnvltf\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d4240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-wnvltf: failed waiting for cluster=\"cp-ystream-upgrade-4-22-wnvltf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gt4769dgnqtg\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4573595a-316c-4f3b-b40c-f401e277ab44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4573595a-316c-4f3b-b40c-f401e277ab44\",\n \"name\": \"4573595a-316c-4f3b-b40c-f401e277ab44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.142641521Z\",\n \"endTime\": \"2026-08-25T11:24:07.734693222Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, hosted-cluster-config-operator, olm-operator, certified-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, community-operators-catalog, cluster-version-operator, cluster-image-registry-operator, packageserver, redhat-marketplace-catalog, openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, redhat-operators-catalog, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-wnvltf\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d4240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-wnvltf: failed waiting for cluster=\"cp-ystream-upgrade-4-22-wnvltf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gt4769dgnqtg\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4573595a-316c-4f3b-b40c-f401e277ab44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4573595a-316c-4f3b-b40c-f401e277ab44\",\n \"name\": \"4573595a-316c-4f3b-b40c-f401e277ab44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.142641521Z\",\n \"endTime\": \"2026-08-25T11:24:07.734693222Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, hosted-cluster-config-operator, olm-operator, certified-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, community-operators-catalog, cluster-version-operator, cluster-image-registry-operator, packageserver, redhat-marketplace-catalog, openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, redhat-operators-catalog, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c8b2a686c9b5a94aaa75ab2862e131ef39a8016f95d0da9b69ad581101fa4e1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"7c0e987f55407504abfc17fba96a678154c2fb6eca07c78b46a987ce3627cfc1","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5ae80\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jpmgv9dccmdw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"name\": \"0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.150172423Z\",\n \"endTime\": \"2026-08-25T11:23:47.559518518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5ae80\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jpmgv9dccmdw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"name\": \"0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.150172423Z\",\n \"endTime\": \"2026-08-25T11:23:47.559518518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7ad14e1158d7d53f17395adc1367d8f6b3116098187de178cccafef487d5b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ca10e452c73ffaa1858f8cb01f814acceea6055a36e48d6eee4965ea43085f04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"e709edfaf5822475a6c3cdc7e438cd913aefedf35c5c4aa2177cb9bc8689a712","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b39a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-g89x2rmfntgq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\",\n \"name\": \"8ed45382-6139-4927-aeb6-b2580f808168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:09.765576241Z\",\n \"endTime\": \"2026-08-25T11:26:17.763320122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b39a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-g89x2rmfntgq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\",\n \"name\": \"8ed45382-6139-4927-aeb6-b2580f808168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:07:09.765576241Z\",\n \"endTime\": \"2026-08-25T11:26:17.763320122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ca4fe827d5d9e1a291d795c14704446e25bd6969578a5908bc5b86cbf0909737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"alert","signature_id":"783b9a11bba8d6f9b48cde87f4a3968184ce14d623eaa09a0d5aae21b1c10281","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 214 time(s) (72 unknown, 142 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:21:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoqoqjvco52utltqalrje33t0l73-r7q7g1t2h6y9g9u\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoqoqjvco52utltqalrje33t0l73-r7q7g1t2h6y9g9u/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:22:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihornf3ggkrokf3iq4ad00tndcp59-n3t7d0y6p7g0y6a\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihornf3ggkrokf3iq4ad00tndcp59-n3t7d0y6p7g0y6a/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:22:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihojlg7t1ptd1385e8guhsjjbk24b-m7y2e5b2p6u7y0j\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihojlg7t1ptd1385e8guhsjjbk24b-m7y2e5b2p6u7y0j/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:21:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:21:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoil30komhjm90bd4i3lf52t0p7e-n4f8u2r9f9j9f6a\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoil30komhjm90bd4i3lf52t0p7e-n4f8u2r9f9j9f6a/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:22:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm6t4u7847ruif0skle6hdqdtfu0-oidc-wi-cluster\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm6t4u7847ruif0skle6hdqdtfu0-oidc-wi-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:21:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T11:15:53Z\n Ended: 2026-08-25T11:22:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm6t4u7847ruif0skle6hdqdtfu0-oidc-wi-cluster\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm6t4u7847ruif0skle6hdqdtfu0-oidc-wi-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T11:16:58Z\n Ended: 2026-08-25T11:22:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm4a8cjib0rtu5naaoqj6fodpb40-basic-cluster\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm4a8cjib0rtu5naaoqj6fodpb40-basic-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T11:16:58Z\n Ended: 2026-08-25T11:22:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoqo22gh0a0hddp22l6hd9jkoifi-c3d6z1s0d6g5c0i\", pod=\"router-88b4485c8-6spc7\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoqo22gh0a0hddp22l6hd9jkoifi-c3d6z1s0d6g5c0i/router-88b4485c8-6spc7 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T11:17:53Z\n Ended: 2026-08-25T11:26:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiico87tbt5b77mctlj7ovpf2bj5rv-cilium-cluster\", pod=\"router-65fcc9bc6d-7cmzc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiico87tbt5b77mctlj7ovpf2bj5rv-cilium-cluster/router-65fcc9bc6d-7cmzc has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T11:17:53Z\n Ended: 2026-08-25T11:23:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihrhpvfule14t15r9pd9emls5ui4e-b3l7a0h4c9y3h5i\", pod=\"router-8475d5578f-kw7rk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihrhpvfule14t15r9pd9emls5ui4e-b3l7a0h4c9y3h5i/router-8475d5578f-kw7rk has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T11:17:53Z\n Ended: 2026-08-25T11:32:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u\", pod=\"router-77b6469bc4-kdxjp\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u/router-77b6469bc4-kdxjp has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T11:18:57Z\n Ended: 2026-08-25T11:24:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihk0jjm46dcuek7fkdv1s0mlma270-b8y5y3n9s1i3e6r\", pod=\"kube-apiserver-684bdb74f4-qrhlb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihk0jjm46dcuek7fkdv1s0mlma270-b8y5y3n9s1i3e6r/kube-apiserver-684bdb74f4-qrhlb has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihjm1mhlr4psccldumle6ua3dk7h1-l8c7m3s0e0m1c0f\", pod=\"kube-apiserver-56bc7995d6-qk57b\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihjm1mhlr4psccldumle6ua3dk7h1-l8c7m3s0e0m1c0f/kube-apiserver-56bc7995d6-qk57b has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihlpvvvkmgro6e6csbku88ncb9en6-s0x4p8y9x0c6v4m\", pod=\"kube-apiserver-b6f44b495-zr7g8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihlpvvvkmgro6e6csbku88ncb9en6-s0x4p8y9x0c6v4m/kube-apiserver-b6f44b495-zr7g8 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiiavncdrhqeuttup2n9hg48qes0ha-g4d1n8t0u3e4d7d\", pod=\"cluster-node-tuning-operator-56ff6789cc-nkt22\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiiavncdrhqeuttup2n9hg48qes0ha-g4d1n8t0u3e4d7d/cluster-node-tuning-operator-56ff6789cc-nkt22 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihk0f5j57g03nmcd0mogmv2dt2nku-z8h9z4y3f3u5t3o\", pod=\"openshift-apiserver-dff87b4bd-g5j9z\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihk0f5j57g03nmcd0mogmv2dt2nku-z8h9z4y3f3u5t3o/openshift-apiserver-dff87b4bd-g5j9z has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihonffubcm8u116hplvg0t738r0a1-j9l0h3i4s3p1o5k\", pod=\"kube-apiserver-6745758fb9-j5pnz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihonffubcm8u116hplvg0t738r0a1-j9l0h3i4s3p1o5k/kube-apiserver-6745758fb9-j5pnz has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihn8kefnu9ef3p1g5eoidqlfh9uch-c5i6v3u9t2p2s5b\", pod=\"openshift-apiserver-b9cdd8b74-llmwt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihn8kefnu9ef3p1g5eoidqlfh9uch-c5i6v3u9t2p2s5b/openshift-apiserver-b9cdd8b74-llmwt has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihooatb2qifmu342ldo11dqubp742-b5f2i5i5a5b3l2r\", pod=\"kube-apiserver-7c8b6d766f-hmccd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihooatb2qifmu342ldo11dqubp742-b5f2i5i5a5b3l2r/kube-apiserver-7c8b6d766f-hmccd has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-25T11:19:55Z\n Ended: 2026-08-25T11:25:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiiavncdrhqeuttup2n9hg48qes0ha-g4d1n8t0u3e4d7d\", pod=\"openshift-controller-manager-7646ffc7c8-988nr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiiavncdrhqeuttup2n9hg48qes0ha-g4d1n8t0u3e4d7d/openshift-controller-manager-7646ffc7c8-988nr has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-08-25T11:20:53Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp5qru6coctlgq10q529qq8q8t3k-s0z6a1q4s4a5y3k\", pod=\"cluster-storage-operator-78ffcb4d4c-lbg55\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp5qru6coctlgq10q529qq8q8t3k-s0z6a1q4s4a5y3k/cluster-storage-operator-78ffcb4d4c-lbg55 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-08-25T11:20:53Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp5qru6coctlgq10q529qq8q8t3k-s0z6a1q4s4a5y3k\", pod=\"dns-operator-f98478b88-dx44g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp5qru6coctlgq10q529qq8q8t3k-s0z6a1q4s4a5y3k/dns-operator-f98478b88-dx44g has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl\", pod=\"ingress-operator-846bbf64f7-v6nwx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl/ingress-operator-846bbf64f7-v6nwx has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl\", pod=\"cluster-network-operator-b897ddf64-4v29h\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl/cluster-network-operator-b897ddf64-4v29h has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl\", pod=\"dns-operator-858b5f4f74-zzxv2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl/dns-operator-858b5f4f74-zzxv2 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihud59ft0p329osbaf1otr9ejbg1p-private-kas\", pod=\"kube-apiserver-5f969fff47-mvbml\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihud59ft0p329osbaf1otr9ejbg1p-private-kas/kube-apiserver-5f969fff47-mvbml has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihjqfseqk0bkameel5g0dcmelgj1p-ea-cluster\", pod=\"dns-operator-576db85c76-7hgp4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihjqfseqk0bkameel5g0dcmelgj1p-ea-cluster/dns-operator-576db85c76-7hgp4 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihk2iajl1nss25pddg85qcnt48735-u9g4x2e7k1c4y9t\", pod=\"azure-cloud-controller-manager-7cb5476c8c-t78kj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihk2iajl1nss25pddg85qcnt48735-u9g4x2e7k1c4y9t/azure-cloud-controller-manager-7cb5476c8c-t78kj has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp5qru6coctlgq10q529qq8q8t3k-s0z6a1q4s4a5y3k\", pod=\"dns-operator-5bc9f885d9-ngvtl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp5qru6coctlgq10q529qq8q8t3k-s0z6a1q4s4a5y3k/dns-operator-5bc9f885d9-ngvtl has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihr2jf10qmgvg3b8sudro0sel69i3-t0x2s4i2j0p9y5k\", pod=\"cluster-node-tuning-operator-7c97fc476c-jrzh2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihr2jf10qmgvg3b8sudro0sel69i3-t0x2s4i2j0p9y5k/cluster-node-tuning-operator-7c97fc476c-jrzh2 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:33:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihr3d1lisej0e8km2sm69rbb5btmv-n1k8p5v8e7s2o2y\", pod=\"router-7647c6448d-mb5j7\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihr3d1lisej0e8km2sm69rbb5btmv-n1k8p5v8e7s2o2y/router-7647c6448d-mb5j7 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl\", pod=\"csi-snapshot-controller-operator-7857d4b9b-dljzv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm0uplesi9d5agopa3d0t0oe12qi-cilium-cl/csi-snapshot-controller-operator-7857d4b9b-dljzv has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-08-25T11:21:57Z\n Ended: 2026-08-25T11:27:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u\", pod=\"azure-cloud-controller-manager-9f68d94cb-n7fvw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u/azure-cloud-controller-manager-9f68d94cb-n7fvw has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-08-25T11:22:56Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihogf52emihtpnapdqhgjjda1t302-g0u4p3s3b9m3g5r\", pod=\"cluster-image-registry-operator-86d46445b-62xt9\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihogf52emihtpnapdqhgjjda1t302-g0u4p3s3b9m3g5r/cluster-image-registry-operator-86d46445b-62xt9 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-08-25T11:22:56Z\n Ended: 2026-08-25T11:33:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoqo22gh0a0hddp22l6hd9jkoifi-c3d6z1s0d6g5c0i\", pod=\"router-785ddc79dd-dvmfd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoqo22gh0a0hddp22l6hd9jkoifi-c3d6z1s0d6g5c0i/router-785ddc79dd-dvmfd has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-08-25T11:22:56Z\n Ended: 2026-08-25T11:29:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihov0r88snbbbcn0aeubuu3akj96i-ephemeral-disk\", pod=\"cluster-storage-operator-79b759d5b6-6bbz6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihov0r88snbbbcn0aeubuu3akj96i-ephemeral-disk/cluster-storage-operator-79b759d5b6-6bbz6 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-08-25T11:22:56Z\n Ended: 2026-08-25T11:29:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihojlg7t1ptd1385e8guhsjjbk24b-m7y2e5b2p6u7y0j\", pod=\"kube-apiserver-c48b9b6f7-jgrct\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihojlg7t1ptd1385e8guhsjjbk24b-m7y2e5b2p6u7y0j/kube-apiserver-c48b9b6f7-jgrct has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-08-25T11:22:56Z\n Ended: 2026-08-25T11:29:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm7cgmbogpmampkrspaq5hon0cos-y1d8u1t5b8e3q7j\", pod=\"kube-apiserver-b987c96f4-cfhp8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm7cgmbogpmampkrspaq5hon0cos-y1d8u1t5b8e3q7j/kube-apiserver-b987c96f4-cfhp8 has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-08-25T11:22:57Z\n Ended: 2026-08-25T11:29:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm7cgmbogpmampkrspaq5hon0cos-y1d8u1t5b8e3q7j\", pod=\"kube-apiserver-b987c96f4-x8snx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm7cgmbogpmampkrspaq5hon0cos-y1d8u1t5b8e3q7j/kube-apiserver-b987c96f4-x8snx has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihjmdk026fo2kbma08qfa1cejc49u-private-ingress\", pod=\"openshift-apiserver-68f6b7c98b-fzttk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihjmdk026fo2kbma08qfa1cejc49u-private-ingress/openshift-apiserver-68f6b7c98b-fzttk has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiibbfs9l2infhclpl8df2iq6lbgu9-k1x8g2z9k7d2n2n\", pod=\"openshift-apiserver-cf6d9d745-jsdt4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiibbfs9l2infhclpl8df2iq6lbgu9-k1x8g2z9k7d2n2n/openshift-apiserver-cf6d9d745-jsdt4 has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp5sbk2173baqaesnoqdga6u4via-v20260901\", pod=\"cluster-storage-operator-776855f9bb-z84fc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp5sbk2173baqaesnoqdga6u4via-v20260901/cluster-storage-operator-776855f9bb-z84fc has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihjmdk026fo2kbma08qfa1cejc49u-private-ingress\", pod=\"machine-approver-587dbd6f84-mgcg2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihjmdk026fo2kbma08qfa1cejc49u-private-ingress/machine-approver-587dbd6f84-mgcg2 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:33:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihrhpvfule14t15r9pd9emls5ui4e-b3l7a0h4c9y3h5i\", pod=\"router-57fff74698-2zzgx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihrhpvfule14t15r9pd9emls5ui4e-b3l7a0h4c9y3h5i/router-57fff74698-2zzgx has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp5sbk2173baqaesnoqdga6u4via-v20260901\", pod=\"csi-snapshot-controller-operator-6ccc765f9c-wqphn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp5sbk2173baqaesnoqdga6u4via-v20260901/csi-snapshot-controller-operator-6ccc765f9c-wqphn has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp5sbk2173baqaesnoqdga6u4via-v20260901\", pod=\"dns-operator-6666fbcc7-jpqbl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp5sbk2173baqaesnoqdga6u4via-v20260901/dns-operator-6666fbcc7-jpqbl has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:32:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihk2iajl1nss25pddg85qcnt48735-u9g4x2e7k1c4y9t\", pod=\"ingress-operator-64d7b6d557-9njjm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihk2iajl1nss25pddg85qcnt48735-u9g4x2e7k1c4y9t/ingress-operator-64d7b6d557-9njjm has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-08-25T11:23:55Z\n Ended: 2026-08-25T11:29:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoqo22gh0a0hddp22l6hd9jkoifi-c3d6z1s0d6g5c0i\", pod=\"dns-operator-648d47796d-2dghs\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoqo22gh0a0hddp22l6hd9jkoifi-c3d6z1s0d6g5c0i/dns-operator-648d47796d-2dghs has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoo74po3sqe3cajn5hji6egsflgt-sre-hcp-cluster\", pod=\"openshift-controller-manager-89cf84d4f-zx7s6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoo74po3sqe3cajn5hji6egsflgt-sre-hcp-cluster/openshift-controller-manager-89cf84d4f-zx7s6 has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u\", pod=\"cluster-node-tuning-operator-c799c98fc-j7mg4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u/cluster-node-tuning-operator-c799c98fc-j7mg4 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:31:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u\", pod=\"dns-operator-c596dc85c-r7kmb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u/dns-operator-c596dc85c-r7kmb has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u\", pod=\"openshift-route-controller-manager-864c948c74-ncqjn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u/openshift-route-controller-manager-864c948c74-ncqjn has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x\", pod=\"cluster-node-tuning-operator-bbb458659-r9hdr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x/cluster-node-tuning-operator-bbb458659-r9hdr has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:31:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u\", pod=\"ingress-operator-5bd9df95c6-l59w6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u/ingress-operator-5bd9df95c6-l59w6 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u\", pod=\"openshift-route-controller-manager-864c948c74-54xlv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u/openshift-route-controller-manager-864c948c74-54xlv has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm7bk5doidrj40qnllvjids6cmu3-q3l8j5u5k3u1a0k\", pod=\"cluster-network-operator-756fb5c44f-pg7cs\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm7bk5doidrj40qnllvjids6cmu3-q3l8j5u5k3u1a0k/cluster-network-operator-756fb5c44f-pg7cs has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp42qokd8iirnheso0hhb5iadhlg-a2x2i0t6e1p5b5y\", pod=\"openshift-controller-manager-f44f79c46-kgcnh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp42qokd8iirnheso0hhb5iadhlg-a2x2i0t6e1p5b5y/openshift-controller-manager-f44f79c46-kgcnh has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-08-25T11:24:54Z\n Ended: 2026-08-25T11:33:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihoo74po3sqe3cajn5hji6egsflgt-sre-hcp-cluster\", pod=\"openshift-route-controller-manager-6b4b5b6f99-4jmj5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihoo74po3sqe3cajn5hji6egsflgt-sre-hcp-cluster/openshift-route-controller-manager-6b4b5b6f99-4jmj5 has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihr3d1lisej0e8km2sm69rbb5btmv-n1k8p5v8e7s2o2y\", pod=\"openshift-route-controller-manager-5d79b9bdf6-hddjw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihr3d1lisej0e8km2sm69rbb5btmv-n1k8p5v8e7s2o2y/openshift-route-controller-manager-5d79b9bdf6-hddjw has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u\", pod=\"cluster-node-tuning-operator-756b5dff7b-h48nf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u/cluster-node-tuning-operator-756b5dff7b-h48nf has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihr3d1lisej0e8km2sm69rbb5btmv-n1k8p5v8e7s2o2y\", pod=\"cluster-policy-controller-854567cccb-cws4c\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihr3d1lisej0e8km2sm69rbb5btmv-n1k8p5v8e7s2o2y/cluster-policy-controller-854567cccb-cws4c has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihn2lfkgh3lt1mq76md59k1m8bn2f-p9u7t7f4z4a6g8h\", pod=\"openshift-controller-manager-6b6546f4f7-qxkjm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihn2lfkgh3lt1mq76md59k1m8bn2f-p9u7t7f4z4a6g8h/openshift-controller-manager-6b6546f4f7-qxkjm has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm7bk5doidrj40qnllvjids6cmu3-q3l8j5u5k3u1a0k\", pod=\"openshift-controller-manager-5db4c96ff7-lscxk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm7bk5doidrj40qnllvjids6cmu3-q3l8j5u5k3u1a0k/openshift-controller-manager-5db4c96ff7-lscxk has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihogqel47t0pk7ir633okvtdrk0a2-h7m7s0l5k5s9d3r\", pod=\"cluster-node-tuning-operator-858564c7-v59hc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihogqel47t0pk7ir633okvtdrk0a2-h7m7s0l5k5s9d3r/cluster-node-tuning-operator-858564c7-v59hc has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u\", pod=\"openshift-controller-manager-688457c87b-dr92t\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihnlofblh7399cq4fbi0l5tqtvvn4-e1w9t6c4k1o3x0u/openshift-controller-manager-688457c87b-dr92t has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x\", pod=\"openshift-controller-manager-5dcdfbfc5c-7wksr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiiat3uo7ueo5ragnuq92ks0dk69c7-n6g1w0h5p4t6v7x/openshift-controller-manager-5dcdfbfc5c-7wksr has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u\", pod=\"cluster-network-operator-698b868fdb-w6q94\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u/cluster-network-operator-698b868fdb-w6q94 has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihm7bk5doidrj40qnllvjids6cmu3-q3l8j5u5k3u1a0k\", pod=\"ingress-operator-9747d4f68-vb8bl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihm7bk5doidrj40qnllvjids6cmu3-q3l8j5u5k3u1a0k/ingress-operator-9747d4f68-vb8bl has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihog7uq0k1ofaet0tpch6hfnpp1m6-j2z2w3q5v7n7q1v\", pod=\"openshift-apiserver-6cb576bd79-gqdlj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihog7uq0k1ofaet0tpch6hfnpp1m6-j2z2w3q5v7n7q1v/openshift-apiserver-6cb576bd79-gqdlj has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-08-25T11:24:55Z\n Ended: 2026-08-25T11:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u\", pod=\"cluster-image-registry-operator-548bb6b86c-dss79\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihp0q2ddeb5fgluh885f3vnifv11v-q8l9x8o4f8k6i4u/cluster-image-registry-operator-548bb6b86c-dss79 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"cb558c079251186882c1d187f8453e6910e797dc927c2c1ec41aabca427e9cde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"a7bdda2906b7a00d3ea96fe2c15e55467a147dabf9881ad88f1b9c2804a5d6af","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019a8a40\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-x45zf5lld2jq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5b946af-4c8b-4b08-8c58-6a22ea330788\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5b946af-4c8b-4b08-8c58-6a22ea330788\",\n \"name\": \"f5b946af-4c8b-4b08-8c58-6a22ea330788\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.32799472Z\",\n \"endTime\": \"2026-08-25T11:24:07.638983129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019a8a40\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-x45zf5lld2jq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5b946af-4c8b-4b08-8c58-6a22ea330788\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5b946af-4c8b-4b08-8c58-6a22ea330788\",\n \"name\": \"f5b946af-4c8b-4b08-8c58-6a22ea330788\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.32799472Z\",\n \"endTime\": \"2026-08-25T11:24:07.638983129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d30128bc0e4e31af60d400f30b8e3dc2b8066c214eb4b8964b6df62343824937","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"7962adc60e87afd58df615180673875e6038c2e6c21e761c06a8e2250c897c04","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cfa540\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6b79jbxxk2jv\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"name\": \"51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.544982989Z\",\n \"endTime\": \"2026-08-25T11:23:47.560554329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-version-operator, cluster-network-operator, olm-operator, cluster-storage-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cfa540\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6b79jbxxk2jv\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"name\": \"51cea064-8c25-47d5-9e4c-98159ead62b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:45.544982989Z\",\n \"endTime\": \"2026-08-25T11:23:47.560554329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-version-operator, cluster-network-operator, olm-operator, cluster-storage-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2f193447f10118c111dd9c9ace812ed0603c456872d7c9913367976c1039abf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"8e4e26a914a87830fbedc93c1a377a82ad141c9089c89b9edbacc2b4d4d5a21f","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116340\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bjqmncjbznvt\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\",\n \"name\": \"64e0d347-b695-455c-b828-66deb96d7452\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.579389204Z\",\n \"endTime\": \"2026-08-25T11:24:07.68094221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, packageserver, cluster-network-operator, ingress-operator, cluster-version-operator, openshift-apiserver, dns-operator, cluster-policy-controller, redhat-marketplace-catalog, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116340\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bjqmncjbznvt\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\",\n \"name\": \"64e0d347-b695-455c-b828-66deb96d7452\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.579389204Z\",\n \"endTime\": \"2026-08-25T11:24:07.68094221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, packageserver, cluster-network-operator, ingress-operator, cluster-version-operator, openshift-apiserver, dns-operator, cluster-policy-controller, redhat-marketplace-catalog, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e3e02d8d9bcf2e45b01530f013eced90315da5b6056f1c1cec7f01001d3c4e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"66ee2372a7d35bafc2a7139acace8a079bd17442b7e81b824c6f13e5ea4be15f","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fa820\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gp4rdpwm7zhd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"name\": \"5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.463575249Z\",\n \"endTime\": \"2026-08-25T11:23:57.522671815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, olm-collect-profiles, cluster-policy-controller, cluster-node-tuning-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, cluster-network-operator, ignition-server-proxy, catalog-operator, openshift-apiserver, ignition-server, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, machine-approver, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, control-plane-pki-operator, kube-scheduler, cluster-version-operator, cluster-autoscaler, openshift-route-controller-manager, azure-cloud-controller-manager, ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fa820\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gp4rdpwm7zhd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"name\": \"5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:55.463575249Z\",\n \"endTime\": \"2026-08-25T11:23:57.522671815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, olm-collect-profiles, cluster-policy-controller, cluster-node-tuning-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, cluster-network-operator, ignition-server-proxy, catalog-operator, openshift-apiserver, ignition-server, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, machine-approver, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, control-plane-pki-operator, kube-scheduler, cluster-version-operator, cluster-autoscaler, openshift-route-controller-manager, azure-cloud-controller-manager, ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2bdf0d0e26486c8baecfb566f0a7ae932a4a2cee72db742fb786cc445dbea21e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, catalog-operator, certified-operators-catalog, cluster-autoscaler, cluster-image-registry-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, cluster-storage-operator, cluster-version-operator, community-operators-catalog, control-plane-pki-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ignition-server, ignition-server-proxy, ingress-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, machine-approver, olm-collect-profiles, olm-operator, openshift-apiserver, openshift-controller-manager, openshift-route-controller-manager, packageserver, redhat-marketplace-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e54d8faa911a3dc82f0a42b9a5abbf2df1d14caaad5e473c58ba7fae0967982d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"62ef36a640529db2357933eec4cfc1865780247b38f5766b5fa5e83338b90158","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001081c80\u003e: \n GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"name\": \"f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:19.502868721Z\",\n \"endTime\": \"2026-08-25T11:25:27.633996529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001081c80\u003e: \n GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"name\": \"f7baee5b-db11-4464-aa8e-6e2d27501249\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:19.502868721Z\",\n \"endTime\": \"2026-08-25T11:25:27.633996529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e339823035a3b42061d546c2e6cbdec3cd57b33bd40fddf5adef300f46092d74","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, openshift-controller-manager; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8949c0c38b793ed66679ca292ecb1d0c2b71f99a8fcf4d1a485d1604a6a239a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"8366f1f3e151dc6dc562cd5621b250932345234159c653b20f33f1095b4d2763","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb08a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-twjmtksnvtqp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83902b43-eb0b-4d5c-8306-2571781dc775\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83902b43-eb0b-4d5c-8306-2571781dc775\",\n \"name\": \"83902b43-eb0b-4d5c-8306-2571781dc775\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.672178796Z\",\n \"endTime\": \"2026-08-25T11:24:07.708626146Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, catalog-operator, packageserver, cluster-node-tuning-operator, cluster-network-operator, cluster-image-registry-operator, dns-operator, ingress-operator, cluster-storage-operator, olm-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb08a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-twjmtksnvtqp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83902b43-eb0b-4d5c-8306-2571781dc775\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83902b43-eb0b-4d5c-8306-2571781dc775\",\n \"name\": \"83902b43-eb0b-4d5c-8306-2571781dc775\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:07.672178796Z\",\n \"endTime\": \"2026-08-25T11:24:07.708626146Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, catalog-operator, packageserver, cluster-node-tuning-operator, cluster-network-operator, cluster-image-registry-operator, dns-operator, ingress-operator, cluster-storage-operator, olm-operator, hosted-cluster-config-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec1ad7aaf8bfb29986a54322078a11627cba93246b561c75168b8e5926c14072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"e70ccddbda0336cf522314d54bd2bb07c61aafcb132de78c1ca60fa8b7fd1d35","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xk9vbv\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000ca0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xk9vbv: failed waiting for cluster=\"cluster-zstream-5-0-xk9vbv\" in resourcegroup=\"rg-zstream-upgrade-5-0-9zdkrc5gfnqq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"name\": \"7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.893167006Z\",\n \"endTime\": \"2026-08-25T11:24:07.716194982Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xk9vbv\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000ca0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xk9vbv: failed waiting for cluster=\"cluster-zstream-5-0-xk9vbv\" in resourcegroup=\"rg-zstream-upgrade-5-0-9zdkrc5gfnqq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"name\": \"7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:05.893167006Z\",\n \"endTime\": \"2026-08-25T11:24:07.716194982Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ac40957e0d202cc6756c5cbbe3d0ef3ac7730a3425dfc91631cac2e91ff702d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f2cd1d850c45aaaf275bc9e454572de7d804564c3b7c33f43e8037c886cf7be0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"baf1c4495aac4315f07ab15d045d6bf7c36c718b7b6f2252cb6b1dd46b38088c","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-wmc2th\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80de0\u003e: \n failed to create HCP cluster np-noedge-wmc2th: failed waiting for cluster=\"np-noedge-wmc2th\" in resourcegroup=\"rg-np-noedge-wmc2th-55l76kt7fdfw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"name\": \"599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.955311835Z\",\n \"endTime\": \"2026-08-25T11:23:47.543149183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, olm-operator, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-wmc2th\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80de0\u003e: \n failed to create HCP cluster np-noedge-wmc2th: failed waiting for cluster=\"np-noedge-wmc2th\" in resourcegroup=\"rg-np-noedge-wmc2th-55l76kt7fdfw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"name\": \"599e72ae-426e-413c-a454-0c456b2c46e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:46.955311835Z\",\n \"endTime\": \"2026-08-25T11:23:47.543149183Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, olm-operator, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7c1668e400aa4b2fc13f39642afb5c95fd77666aaa09a045bad1fcfaedb6d3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","failed_at":"e2e","signature_id":"e3135c78719260f8b136dccd2d27e839bcb88513840553199c7a5a72c9c78af4","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001095660\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-b4qlj57lcxr8\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"name\": \"c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:28.224401746Z\",\n \"endTime\": \"2026-08-25T11:24:37.557730725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, olm-operator, packageserver, redhat-operators-catalog, dns-operator, ingress-operator, cluster-version-operator, openshift-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001095660\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-b4qlj57lcxr8\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"name\": \"c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:28.224401746Z\",\n \"endTime\": \"2026-08-25T11:24:37.557730725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, olm-operator, packageserver, redhat-operators-catalog, dns-operator, ingress-operator, cluster-version-operator, openshift-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7bb05c760ce6e02186100a06d53cbce3c5ec829fc4f0c3e2e6bbbb1e0d7f73e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-version-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":52,"failure_pattern_ids":["02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","183ac5e7e1c7f43ad252fd54e6e3fae5ee8421fc47486bdad5d091cff0dbf9a9","1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","279b8dc059ffca13ba2e7632bee0bb38b6b6e79fa3da4d820708453f05c04b29","2bdf0d0e26486c8baecfb566f0a7ae932a4a2cee72db742fb786cc445dbea21e","39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","470aab7d7a3605e8633a637a70d37da16684c6630946576af6d433f851290640","55a75f995426be3c05299ad4947251679d4a99ba7e77d726aeadaba9bd31ae5d","56ac1ffd8f7c84d6800647347b139a137e691a55ff2e3d1faa305a6b07101364","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","75a2ddc5380cf22cdc0f7eee734c0875c6030bb3df880d9c39c49bb360b07d00","7be6b09c6bbb28adba99c6de7dfc1604dfce000bc6664dedbae8e8b9b0253fa6","8172b3b938ddc92e532decd443ed82e33579c57cbdab74ca3567337de4aab992","8a4546d1ccb11eb2bd205f13e52f18ff56616b802ad979a8ada123b2e8ac5a07","8fcfad7ee636c64bcc7ec16c2a0da480eb90b3d648cf1239ba480534695bb74a","92ae44f2e86af5544dec422ded7c331d2f560f22fa4954538768a565c71a9dbd","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9917e0e4cec099cb18cbbcab5a48ffe13a11b43e10f38323888849ed9a27dd49","9ac40957e0d202cc6756c5cbbe3d0ef3ac7730a3425dfc91631cac2e91ff702d","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","a7bb05c760ce6e02186100a06d53cbce3c5ec829fc4f0c3e2e6bbbb1e0d7f73e","b393a3822cea98a3f0355403abde765ef631da69f047c1828358b6194172cb6b","b3d328e9a97f3f976227e1f892def10b7f6c53bda848b798f9760113f189d2a4","b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","c4fdcee2df11b558d3b3164768a55e10395e8648b1940e20c553175bae5d91ab","c975965241af31c57ec44c3fff1102f75c167bb738954ccc985f5a6bb7a81cd2","da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","e339823035a3b42061d546c2e6cbdec3cd57b33bd40fddf5adef300f46092d74","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","f59d41db8c7110a2a66df7a04a929104d4f74b17a121064979556d85ac966c14","f7ad14e1158d7d53f17395adc1367d8f6b3116098187de178cccafef487d5b29"],"matched_occurrences":52,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:02:57Z"},"failed_at":["alert","e2e"],"failed_test_count":40,"occurrences":[{"row_id":"05ca6a95b3d94f1a2f7591593cf2bebe313fb4c473cb0a27ea4ea2922db4c25d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"0ac0ed3b0a9732a181325ecdaa144e0e29914071fc6f1496b3b0c6f22509cdba","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e500\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kxgdkssgfc6d\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8f655cf-cd5c-4298-b821-2efa60072edd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8f655cf-cd5c-4298-b821-2efa60072edd\",\n \"name\": \"c8f655cf-cd5c-4298-b821-2efa60072edd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:18:37.705265753Z\",\n \"endTime\": \"2026-08-25T11:37:45.81706187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e500\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kxgdkssgfc6d\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8f655cf-cd5c-4298-b821-2efa60072edd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8f655cf-cd5c-4298-b821-2efa60072edd\",\n \"name\": \"c8f655cf-cd5c-4298-b821-2efa60072edd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:18:37.705265753Z\",\n \"endTime\": \"2026-08-25T11:37:45.81706187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0994542d709383fdcd136d0a72110f95d47a8062ebb359b6861ab4bfc48238d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"01c47822900529458c20e60551800d6a17821b995d4851679b62a387c6efaa2c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0b0e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-57kmhshxtcx2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"name\": \"b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:19.928553979Z\",\n \"endTime\": \"2026-08-25T11:21:25.9465048Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-storage-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-image-registry-operator, packageserver, olm-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0b0e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-57kmhshxtcx2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"name\": \"b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:19.928553979Z\",\n \"endTime\": \"2026-08-25T11:21:25.9465048Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-storage-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-image-registry-operator, packageserver, olm-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1264ce1f65fb1c4e4b03ef0f0b493117b1b9becb99a5e03d914e5ae33ea84790","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b9c8a9a2229a849e8a846a577ab549784fe31efea83ec8ab492686f05cb6abd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"7e3686375be61c20eb803573e751563f0941ff5159af2cf0a463b90d62b70143","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f761c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tsn8fjg7z8v8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"name\": \"7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.548851412Z\",\n \"endTime\": \"2026-08-25T11:21:35.946063021Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-version-operator, openshift-route-controller-manager, olm-operator, packageserver, certified-operators-catalog, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, redhat-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f761c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tsn8fjg7z8v8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"name\": \"7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.548851412Z\",\n \"endTime\": \"2026-08-25T11:21:35.946063021Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-version-operator, openshift-route-controller-manager, olm-operator, packageserver, certified-operators-catalog, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, redhat-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0d07f3cd480569d4660dd225254881834ff9b30edc1bae32dc9a6054119c5e31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"84e02a1a3635ca3e59119a735efac8221692eec41952c2200fb85b25108b853d","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce01c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vm8pwb95swzd\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"name\": \"17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:11.32352815Z\",\n \"endTime\": \"2026-08-25T11:22:15.832343372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-image-registry-operator, dns-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog, openshift-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-apiserver, ingress-operator, olm-operator, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce01c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vm8pwb95swzd\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"name\": \"17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:11.32352815Z\",\n \"endTime\": \"2026-08-25T11:22:15.832343372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-image-registry-operator, dns-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog, openshift-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-apiserver, ingress-operator, olm-operator, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6200cb8cea81a44cbffdcbf30f49e1b2e8791793e7e07d52640fca8ceb6c92c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"12c48630bd31884b8c43bf1ce8575b8b25fd34b2eb4481d081be350cafa6cf7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"84df8ea31194f1a34ce54d507db6d896e44775caf63f19fedd4bd03f18b43d1b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e81d20\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-wtmf95jcsb5k\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14e367e4-ea49-470a-a267-1aa8a5819f26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14e367e4-ea49-470a-a267-1aa8a5819f26\",\n \"name\": \"14e367e4-ea49-470a-a267-1aa8a5819f26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:32.694001765Z\",\n \"endTime\": \"2026-08-25T11:23:35.873645251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-version-operator, cluster-network-operator, redhat-marketplace-catalog, community-operators-catalog, ingress-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, openshift-controller-manager, dns-operator, cluster-node-tuning-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 3 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 3 unavailable replicas, openshift-route-controller-manager deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e81d20\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-wtmf95jcsb5k\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14e367e4-ea49-470a-a267-1aa8a5819f26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14e367e4-ea49-470a-a267-1aa8a5819f26\",\n \"name\": \"14e367e4-ea49-470a-a267-1aa8a5819f26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:32.694001765Z\",\n \"endTime\": \"2026-08-25T11:23:35.873645251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-version-operator, cluster-network-operator, redhat-marketplace-catalog, community-operators-catalog, ingress-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, openshift-controller-manager, dns-operator, cluster-node-tuning-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 3 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 3 unavailable replicas, openshift-route-controller-manager deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1e2165fa8ad5af3155f607105457313aa908c1716752c66b60be54bdd50a4479","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"66a0672a0448bcbf1d60a1d5cafd7b371b74508101d9fc34d0777b13e4cf76e2","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00021f7c0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8s5smmwtzc77\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b32196f2-1fcd-4826-ae2c-d7a59f640453\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b32196f2-1fcd-4826-ae2c-d7a59f640453\",\n \"name\": \"b32196f2-1fcd-4826-ae2c-d7a59f640453\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.79442888Z\",\n \"endTime\": \"2026-08-25T11:21:35.920091579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00021f7c0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8s5smmwtzc77\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b32196f2-1fcd-4826-ae2c-d7a59f640453\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b32196f2-1fcd-4826-ae2c-d7a59f640453\",\n \"name\": \"b32196f2-1fcd-4826-ae2c-d7a59f640453\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.79442888Z\",\n \"endTime\": \"2026-08-25T11:21:35.920091579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1f6a6b0b9d7d8fd8b6fd4ffad1958d0b0967b427acf8812c8a4553b87c85ee8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"1e1bd91e2da41c37dab2549990d0b6c3580daa21ab6ad95e1315e6a8a8df6173","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880340\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-fncq4c5d429d\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7409d60a-0ba9-4daa-99e6-fd3d757217a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7409d60a-0ba9-4daa-99e6-fd3d757217a6\",\n \"name\": \"7409d60a-0ba9-4daa-99e6-fd3d757217a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:21.864024356Z\",\n \"endTime\": \"2026-08-25T11:23:25.850925072Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, catalog-operator, cluster-network-operator, olm-operator, cluster-version-operator, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator, packageserver, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880340\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-fncq4c5d429d\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7409d60a-0ba9-4daa-99e6-fd3d757217a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7409d60a-0ba9-4daa-99e6-fd3d757217a6\",\n \"name\": \"7409d60a-0ba9-4daa-99e6-fd3d757217a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:21.864024356Z\",\n \"endTime\": \"2026-08-25T11:23:25.850925072Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, catalog-operator, cluster-network-operator, olm-operator, cluster-version-operator, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator, packageserver, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"26faeb519ce3141fc203bc9ce72dba9ee1b94556fad4ca753e4683cb843156b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"4b8149b33710d3d2d55931ca7538e774a86bcfc7ef3d236c36f5ffb2ecb5845f","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7a340\u003e: \n failed to create HCP cluster np-dg-minor-5k4t7m: failed waiting for cluster=\"np-dg-minor-5k4t7m\" in resourcegroup=\"rg-np-dg-minor-5k4t7m-c7rmk8jp25j5\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"name\": \"36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.5276884Z\",\n \"endTime\": \"2026-08-25T11:21:35.939284741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7a340\u003e: \n failed to create HCP cluster np-dg-minor-5k4t7m: failed waiting for cluster=\"np-dg-minor-5k4t7m\" in resourcegroup=\"rg-np-dg-minor-5k4t7m-c7rmk8jp25j5\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"name\": \"36c411c9-a482-476f-8c6d-e111f54a5519\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.5276884Z\",\n \"endTime\": \"2026-08-25T11:21:35.939284741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b67b2954ed80e23d5dbe844cbe69ddf2aee82b77fbb4d9b923a377beae494cff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"35561ae2da178e195c892e3b36abadf969fd8fdfc8bca1f0c4b975bc479d13d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"d0e594c3105542ca3fc8d2a4997071027f9acfcf8cc74b3eb6724ac1ff51a864","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018a0760\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-k2ffghkn9d59\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cdfa7a3-a70a-44de-9fde-fd3953477c72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cdfa7a3-a70a-44de-9fde-fd3953477c72\",\n \"name\": \"9cdfa7a3-a70a-44de-9fde-fd3953477c72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:38.97665824Z\",\n \"endTime\": \"2026-08-25T11:21:45.876490122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018a0760\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-k2ffghkn9d59\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cdfa7a3-a70a-44de-9fde-fd3953477c72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cdfa7a3-a70a-44de-9fde-fd3953477c72\",\n \"name\": \"9cdfa7a3-a70a-44de-9fde-fd3953477c72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:38.97665824Z\",\n \"endTime\": \"2026-08-25T11:21:45.876490122Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b6fa9c84b0c134b17ebc89326a5da662117067deda72a70a438112a533ce1b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"alert","signature_id":"c4c245d9532eb880b4ccd3795a4e4bdc7b51cc2471678008cfc9c24b079969f8","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:52:38Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j7671680-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7671680/providers/microsoft.monitor/accounts/services-j7671680\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"46e2e80bf9941f6f89a60722a0f006712c99857442f53e4a3d2454f7932edeee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"29bc2f93a6094b4a5d5cf641b428320d4890242a1af692aabdd89032ec92bca7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wgqxxx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f36680\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wgqxxx: failed waiting for cluster=\"cluster-zstream-5-0-wgqxxx\" in resourcegroup=\"rg-zstream-upgrade-5-0-dtrxh9srjttj\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e03a2cc-036c-400d-8fc0-f3fe60b7e8f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e03a2cc-036c-400d-8fc0-f3fe60b7e8f8\",\n \"name\": \"2e03a2cc-036c-400d-8fc0-f3fe60b7e8f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:05.144488342Z\",\n \"endTime\": \"2026-08-25T11:21:05.810048041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wgqxxx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f36680\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wgqxxx: failed waiting for cluster=\"cluster-zstream-5-0-wgqxxx\" in resourcegroup=\"rg-zstream-upgrade-5-0-dtrxh9srjttj\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e03a2cc-036c-400d-8fc0-f3fe60b7e8f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e03a2cc-036c-400d-8fc0-f3fe60b7e8f8\",\n \"name\": \"2e03a2cc-036c-400d-8fc0-f3fe60b7e8f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:05.144488342Z\",\n \"endTime\": \"2026-08-25T11:21:05.810048041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4df008253c24ee72c5a6ed7486925b2d855e200656c3ba56e54987155d2f61e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"8535cc4178a95a0eda0b65492ec9ecf1ca1fb3239efd1f0b829eea75cb13b88d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c2f59xczgxh8/cluster-candidate-4-21-57drd2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001bba960\u003e: \n failed to create HCP cluster cluster-candidate-4-21-57drd2: failed waiting for cluster=\"cluster-candidate-4-21-57drd2\" in resourcegroup=\"rg-candidate-4-21-c2f59xczgxh8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"name\": \"d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:18.423236357Z\",\n \"endTime\": \"2026-08-25T11:21:25.878162329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c2f59xczgxh8/cluster-candidate-4-21-57drd2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001bba960\u003e: \n failed to create HCP cluster cluster-candidate-4-21-57drd2: failed waiting for cluster=\"cluster-candidate-4-21-57drd2\" in resourcegroup=\"rg-candidate-4-21-c2f59xczgxh8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"name\": \"d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:18.423236357Z\",\n \"endTime\": \"2026-08-25T11:21:25.878162329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5023490616bf17026c3465b6991287cc3fa6d9bf585f36be6e826419c88d4be8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"09789f559d5df6a2a46fb75a6ae925fba0fbd303143b135e8e8bf18f560ae90c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-d4q7n8\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00123e240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-d4q7n8: failed waiting for cluster=\"cp-ystream-upgrade-4-22-d4q7n8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-skn4bzscn96c\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f32701d7-6200-468b-980b-66f6b8257b49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f32701d7-6200-468b-980b-66f6b8257b49\",\n \"name\": \"f32701d7-6200-468b-980b-66f6b8257b49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.254710411Z\",\n \"endTime\": \"2026-08-25T11:21:35.961594722Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-d4q7n8\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00123e240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-d4q7n8: failed waiting for cluster=\"cp-ystream-upgrade-4-22-d4q7n8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-skn4bzscn96c\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f32701d7-6200-468b-980b-66f6b8257b49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f32701d7-6200-468b-980b-66f6b8257b49\",\n \"name\": \"f32701d7-6200-468b-980b-66f6b8257b49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.254710411Z\",\n \"endTime\": \"2026-08-25T11:21:35.961594722Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"51d644cd10daf303126a72cdac39d3df109b98660b7a6d7095b916328cc4dee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"541a9bf4087000257fc777c8bd4137d002f9e23cd4994e52825872d346f25bdb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zgp8947jk9nr/cluster-candidate-4-22-8d2tgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e64200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8d2tgf: failed waiting for cluster=\"cluster-candidate-4-22-8d2tgf\" in resourcegroup=\"rg-candidate-4-22-zgp8947jk9nr\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"name\": \"8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.272254875Z\",\n \"endTime\": \"2026-08-25T11:21:35.930400552Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zgp8947jk9nr/cluster-candidate-4-22-8d2tgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e64200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-8d2tgf: failed waiting for cluster=\"cluster-candidate-4-22-8d2tgf\" in resourcegroup=\"rg-candidate-4-22-zgp8947jk9nr\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"name\": \"8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.272254875Z\",\n \"endTime\": \"2026-08-25T11:21:35.930400552Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"596944682eeb151e84b43b42939ac7cc816a9bd04f295204a6779532da1c4157","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"alert","signature_id":"6791742669eeec9806dabaab1695d3466c0bad5f4d644012bab5a19c0d345acd","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 171 time(s) (64 unknown, 107 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:14:21Z\n Ended: 2026-08-25T11:24:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi4300sn46vldo7trcc7l8m33gfu-j4j6s0z7p6o8r0o\", pod=\"router-86868f6b87-mncnj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi4300sn46vldo7trcc7l8m33gfu-j4j6s0z7p6o8r0o/router-86868f6b87-mncnj has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:14:21Z\n Ended: 2026-08-25T11:20:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigice0hl86i4gt0ludsc58uuieghd-a3m6i0h0h5c7u0t\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigice0hl86i4gt0ludsc58uuieghd-a3m6i0h0h5c7u0t/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigbl78sli3h6n1b1qhu15o6iecgnp-s2e2g8t3k3c7g2l\", pod=\"cluster-network-operator-5b5ddf888-fcvfd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigbl78sli3h6n1b1qhu15o6iecgnp-s2e2g8t3k3c7g2l/cluster-network-operator-5b5ddf888-fcvfd has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighajm87paq4cs16pl3ffhosedk96-ea-cluster\", pod=\"kube-apiserver-85c4d7c5b8-n6zkk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighajm87paq4cs16pl3ffhosedk96-ea-cluster/kube-apiserver-85c4d7c5b8-n6zkk has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigbl78sli3h6n1b1qhu15o6iecgnp-s2e2g8t3k3c7g2l\", pod=\"ingress-operator-5867b47f6c-q64d2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigbl78sli3h6n1b1qhu15o6iecgnp-s2e2g8t3k3c7g2l/ingress-operator-5867b47f6c-q64d2 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigbl78sli3h6n1b1qhu15o6iecgnp-s2e2g8t3k3c7g2l\", pod=\"cluster-image-registry-operator-7d55f546b4-98b6x\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigbl78sli3h6n1b1qhu15o6iecgnp-s2e2g8t3k3c7g2l/cluster-image-registry-operator-7d55f546b4-98b6x has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighp1s7dtgkvt0kn06eru5qhi6hva-i1o3b3q7e1n1e9n\", pod=\"control-plane-operator-5695cc6b8b-cnw6k\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighp1s7dtgkvt0kn06eru5qhi6hva-i1o3b3q7e1n1e9n/control-plane-operator-5695cc6b8b-cnw6k has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o\", pod=\"ignition-server-proxy-86df6fdcc8-9k749\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o/ignition-server-proxy-86df6fdcc8-9k749 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighajm87paq4cs16pl3ffhosedk96-ea-cluster\", pod=\"kube-apiserver-85c4d7c5b8-dgxnd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighajm87paq4cs16pl3ffhosedk96-ea-cluster/kube-apiserver-85c4d7c5b8-dgxnd has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigerd9bfd6ciq1hgspbit0kf6oq8j-j3m9k1l7x1h7s8o\", pod=\"cluster-network-operator-7d4d696f5c-d7m5f\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigerd9bfd6ciq1hgspbit0kf6oq8j-j3m9k1l7x1h7s8o/cluster-network-operator-7d4d696f5c-d7m5f has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o\", pod=\"ignition-server-proxy-86df6fdcc8-pvrr5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o/ignition-server-proxy-86df6fdcc8-pvrr5 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi5nvbb34o4q0i19jdfn9ue3fcf5-c5c3h8c1c5g5f0i\", pod=\"azure-file-csi-driver-operator-865777d9b5-s4lrn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi5nvbb34o4q0i19jdfn9ue3fcf5-c5c3h8c1c5g5f0i/azure-file-csi-driver-operator-865777d9b5-s4lrn has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigkfrvlqkgih9nbm82vtk3rdssfk1-w6r1m5h2r9a2f8g\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigkfrvlqkgih9nbm82vtk3rdssfk1-w6r1m5h2r9a2f8g/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig83ue3dh1bfid9jod5bhu1ujp45b-p8k1j3s5d6k3r0m\", pod=\"ingress-operator-5497d8c5f9-976gs\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig83ue3dh1bfid9jod5bhu1ujp45b-p8k1j3s5d6k3r0m/ingress-operator-5497d8c5f9-976gs has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T11:15:21Z\n Ended: 2026-08-25T11:21:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigerd9bfd6ciq1hgspbit0kf6oq8j-j3m9k1l7x1h7s8o\", pod=\"ingress-operator-6b756d9899-tb648\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigerd9bfd6ciq1hgspbit0kf6oq8j-j3m9k1l7x1h7s8o/ingress-operator-6b756d9899-tb648 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T11:16:25Z\n Ended: 2026-08-25T11:27:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihgagmb9aar4uvjhrdttppcgbm57r-agg-cluster\", pod=\"router-64c45976b6-shhnn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihgagmb9aar4uvjhrdttppcgbm57r-agg-cluster/router-64c45976b6-shhnn has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T11:16:25Z\n Ended: 2026-08-25T11:23:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighba6mofj9027jck00rnhf6mnq12-private-ingress\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighba6mofj9027jck00rnhf6mnq12-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T11:16:25Z\n Ended: 2026-08-25T11:23:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighbsvc7e80p8g1avr31qr63b8u4e-i6s9l5o7e5m7d5g\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighbsvc7e80p8g1avr31qr63b8u4e-i6s9l5o7e5m7d5g/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T11:18:21Z\n Ended: 2026-08-25T11:24:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigs7u0nv00mi79g4kuklicavbrvgq-private-kas\", pod=\"kube-apiserver-7956c4d7f6-nlcp2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigs7u0nv00mi79g4kuklicavbrvgq-private-kas/kube-apiserver-7956c4d7f6-nlcp2 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-25T11:18:21Z\n Ended: 2026-08-25T11:24:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigf6q9jsges99tjc5l5b6vs39n2g0-u7k9x9d2b5o4d7r\", pod=\"kube-apiserver-687b9446fb-5rwrw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigf6q9jsges99tjc5l5b6vs39n2g0-u7k9x9d2b5o4d7r/kube-apiserver-687b9446fb-5rwrw has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w\", pod=\"cluster-storage-operator-69dbd7db4-m5ws6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w/cluster-storage-operator-69dbd7db4-m5ws6 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w\", pod=\"cluster-network-operator-6b85cd7c9c-d5xk4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w/cluster-network-operator-6b85cd7c9c-d5xk4 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi03ntvea6e3k666eglrm2rhp8id-a2j3u3u2c0q7j5k\", pod=\"openshift-apiserver-7db9b89d69-hfnln\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi03ntvea6e3k666eglrm2rhp8id-a2j3u3u2c0q7j5k/openshift-apiserver-7db9b89d69-hfnln has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi4topsd1i9raonsubleo01qb8v0-s9s5k7m8c5x0v2w\", pod=\"kube-apiserver-dd9f6c5b7-mrfx2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi4topsd1i9raonsubleo01qb8v0-s9s5k7m8c5x0v2w/kube-apiserver-dd9f6c5b7-mrfx2 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o\", pod=\"ingress-operator-57bc4cb566-fw8xn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o/ingress-operator-57bc4cb566-fw8xn has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w\", pod=\"ingress-operator-54db749b98-bcptt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w/ingress-operator-54db749b98-bcptt has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi2ogas5reaaof7a0srv70jh6hcg-i4d4z5o5t5w4v5h\", pod=\"cluster-node-tuning-operator-57c96d8565-4s2lb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi2ogas5reaaof7a0srv70jh6hcg-i4d4z5o5t5w4v5h/cluster-node-tuning-operator-57c96d8565-4s2lb has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w\", pod=\"dns-operator-6dd8f6d4dd-g5htt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w/dns-operator-6dd8f6d4dd-g5htt has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigs06ckfrn13g1sgge0jvf8sbu0km-l4d8t0p1v0r2j1o\", pod=\"ingress-operator-5c8fb4fbc-6gbrj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigs06ckfrn13g1sgge0jvf8sbu0km-l4d8t0p1v0r2j1o/ingress-operator-5c8fb4fbc-6gbrj has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigs06ckfrn13g1sgge0jvf8sbu0km-l4d8t0p1v0r2j1o\", pod=\"cluster-network-operator-7694545bf9-xvbhj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigs06ckfrn13g1sgge0jvf8sbu0km-l4d8t0p1v0r2j1o/cluster-network-operator-7694545bf9-xvbhj has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighfolqpd0u9ue3p5c8oft0fg0naq-u5u5r9j6b0w4o1c\", pod=\"kube-apiserver-58bcb897b5-mpp8b\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighfolqpd0u9ue3p5c8oft0fg0naq-u5u5r9j6b0w4o1c/kube-apiserver-58bcb897b5-mpp8b has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigs7u0nv00mi79g4kuklicavbrvgq-private-kas\", pod=\"kube-apiserver-7956c4d7f6-rnw6x\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigs7u0nv00mi79g4kuklicavbrvgq-private-kas/kube-apiserver-7956c4d7f6-rnw6x has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighfolqpd0u9ue3p5c8oft0fg0naq-u5u5r9j6b0w4o1c\", pod=\"kube-apiserver-58bcb897b5-b5rv2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighfolqpd0u9ue3p5c8oft0fg0naq-u5u5r9j6b0w4o1c/kube-apiserver-58bcb897b5-b5rv2 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w\", pod=\"csi-snapshot-controller-operator-76d8c66697-2t55l\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w/csi-snapshot-controller-operator-76d8c66697-2t55l has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighcuhd09i09ka2fr8ik662vk6p1p-l2f1w1h5n1u1u8d\", pod=\"cluster-node-tuning-operator-6b7b576bcc-8s87w\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighcuhd09i09ka2fr8ik662vk6p1p-l2f1w1h5n1u1u8d/cluster-node-tuning-operator-6b7b576bcc-8s87w has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighgebj3n8mr39p8ug1auobc5neu0-a1z4t3b4e1i0s3n\", pod=\"kube-apiserver-6c65d59b9b-bv8wx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighgebj3n8mr39p8ug1auobc5neu0-a1z4t3b4e1i0s3n/kube-apiserver-6c65d59b9b-bv8wx has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:25:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w\", pod=\"dns-operator-867bdbcbc7-gsck8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjt9hhq647c43sc383r5b05542gn-l9o9j4x2m5k3h8w/dns-operator-867bdbcbc7-gsck8 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-08-25T11:18:22Z\n Ended: 2026-08-25T11:24:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighoenag5iarnua2ngld4t6dkitqi-t5a4k0j9i8x9d9q\", pod=\"azure-disk-csi-driver-operator-bf5855b64-dpvgt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighoenag5iarnua2ngld4t6dkitqi-t5a4k0j9i8x9d9q/azure-disk-csi-driver-operator-bf5855b64-dpvgt has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-08-25T11:19:23Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigc9te5976c20animm5nnia7ekeu3-b2f3v8g3z3c8o7r\", pod=\"cluster-storage-operator-7846d67b59-696gz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigc9te5976c20animm5nnia7ekeu3-b2f3v8g3z3c8o7r/cluster-storage-operator-7846d67b59-696gz has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-08-25T11:20:28Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi2i4uc1bn8g630if9mr89dd93s6-d3t6h9z1s7g1y0f\", pod=\"multus-admission-controller-574bffdb8c-kkstx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi2i4uc1bn8g630if9mr89dd93s6-d3t6h9z1s7g1y0f/multus-admission-controller-574bffdb8c-kkstx has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-08-25T11:20:28Z\n Ended: 2026-08-25T11:26:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j\", pod=\"csi-snapshot-controller-operator-9898df5f8-85nl8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j/csi-snapshot-controller-operator-9898df5f8-85nl8 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-08-25T11:20:29Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j\", pod=\"cluster-network-operator-6d4cf88c9b-n8bgz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j/cluster-network-operator-6d4cf88c9b-n8bgz has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-08-25T11:20:29Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig48084ekd4kplpnejldrtd0nto19-f1e9h5t4b7k4k4y\", pod=\"azure-file-csi-driver-operator-76c96977dd-jn44m\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig48084ekd4kplpnejldrtd0nto19-f1e9h5t4b7k4k4y/azure-file-csi-driver-operator-76c96977dd-jn44m has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-08-25T11:20:29Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j\", pod=\"cluster-storage-operator-db6c8d49f-wmv4z\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j/cluster-storage-operator-db6c8d49f-wmv4z has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-08-25T11:20:29Z\n Ended: 2026-08-25T11:28:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j\", pod=\"dns-operator-b65956b94-mj9mm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j/dns-operator-b65956b94-mj9mm has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-08-25T11:20:29Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j\", pod=\"ingress-operator-65bd594575-kvfcs\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j/ingress-operator-65bd594575-kvfcs has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-08-25T11:20:29Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j\", pod=\"cluster-network-operator-6f4b4758c8-mjgbb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j/cluster-network-operator-6f4b4758c8-mjgbb has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-08-25T11:20:30Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihgagmb9aar4uvjhrdttppcgbm57r-agg-cluster\", pod=\"kube-apiserver-85774cfd49-vww6s\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihgagmb9aar4uvjhrdttppcgbm57r-agg-cluster/kube-apiserver-85774cfd49-vww6s has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-08-25T11:20:30Z\n Ended: 2026-08-25T11:26:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j\", pod=\"csi-snapshot-controller-operator-546cfbb8f6-ffvf4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j/csi-snapshot-controller-operator-546cfbb8f6-ffvf4 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-08-25T11:20:30Z\n Ended: 2026-08-25T11:26:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j\", pod=\"ingress-operator-59c85d587f-259t9\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j/ingress-operator-59c85d587f-259t9 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-08-25T11:20:30Z\n Ended: 2026-08-25T11:28:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j\", pod=\"dns-operator-5444f8f58f-hd42v\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8qgenu1ob3k4b4kk5gk82um9oj-m6k9r6v1p6q1p7j/dns-operator-5444f8f58f-hd42v has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-08-25T11:20:30Z\n Ended: 2026-08-25T11:26:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih6k5hmsfknhrbnodu51mipr5pjjn-j1v9x5l4l2q6n5w\", pod=\"openshift-apiserver-57b57cd77b-pwtw9\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih6k5hmsfknhrbnodu51mipr5pjjn-j1v9x5l4l2q6n5w/openshift-apiserver-57b57cd77b-pwtw9 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-08-25T11:20:30Z\n Ended: 2026-08-25T11:27:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j\", pod=\"cluster-image-registry-operator-f66f8556b-xss96\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigmm6ee0hiei23k7aat3sbvnldkgj-z9d9z6o4o8q1n7j/cluster-image-registry-operator-f66f8556b-xss96 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-08-25T11:21:26Z\n Ended: 2026-08-25T11:40:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighp1s7dtgkvt0kn06eru5qhi6hva-i1o3b3q7e1n1e9n\", pod=\"router-69dfd7cb55-k56nw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighp1s7dtgkvt0kn06eru5qhi6hva-i1o3b3q7e1n1e9n/router-69dfd7cb55-k56nw has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-08-25T11:22:23Z\n Ended: 2026-08-25T11:28:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigkfrvlqkgih9nbm82vtk3rdssfk1-w6r1m5h2r9a2f8g\", pod=\"kube-apiserver-6f78b68b96-b8d5v\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigkfrvlqkgih9nbm82vtk3rdssfk1-w6r1m5h2r9a2f8g/kube-apiserver-6f78b68b96-b8d5v has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-08-25T11:22:23Z\n Ended: 2026-08-25T11:28:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigi79la932p945fbgrd10a8k2fh8c-m5x1c7l0g1d8v5m\", pod=\"cluster-policy-controller-685d7d6d55-tldwz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigi79la932p945fbgrd10a8k2fh8c-m5x1c7l0g1d8v5m/cluster-policy-controller-685d7d6d55-tldwz has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:29:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighba6mofj9027jck00rnhf6mnq12-private-ingress\", pod=\"openshift-apiserver-6cd99b5b64-jbnrp\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighba6mofj9027jck00rnhf6mnq12-private-ingress/openshift-apiserver-6cd99b5b64-jbnrp has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:42:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t\", pod=\"cluster-network-operator-6cbf47c7f6-xthqx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t/cluster-network-operator-6cbf47c7f6-xthqx has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:42:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t\", pod=\"ingress-operator-7766cf748d-t6jch\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t/ingress-operator-7766cf748d-t6jch has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:29:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t\", pod=\"cluster-storage-operator-5bd7d48b77-2tpxm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t/cluster-storage-operator-5bd7d48b77-2tpxm has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:42:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t\", pod=\"dns-operator-54f7c7f776-qtchb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t/dns-operator-54f7c7f776-qtchb has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:29:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t\", pod=\"cluster-version-operator-684b54f49-xdd4x\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t/cluster-version-operator-684b54f49-xdd4x has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-08-25T11:23:20Z\n Ended: 2026-08-25T11:29:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o\", pod=\"multus-admission-controller-5599cbcc84-bxmkb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighkj14umms50s1eeb2t0fvtf94n4-z4c8u3h9v6j9o9o/multus-admission-controller-5599cbcc84-bxmkb has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-08-25T11:23:21Z\n Ended: 2026-08-25T11:29:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7671680-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t\", pod=\"csi-snapshot-controller-operator-544df6f47d-rbdp8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighave7ee0g1pk0fk8veq8092mk18-f1e9k5d8w7u9q2t/csi-snapshot-controller-operator-544df6f47d-rbdp8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5ce3b984847922abd33a55d722fb147270eaf5fad38f277a0cff3a906ecadd3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"cc82f7ed850ed911e19f3f8933725fdb7d487b5ae31fdcadcf8ffd1df92eee0f","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4a700\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-4kwl7rkmbbsz\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55b4601f-fd16-4b14-83df-ce42375746d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55b4601f-fd16-4b14-83df-ce42375746d5\",\n \"name\": \"55b4601f-fd16-4b14-83df-ce42375746d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.796667754Z\",\n \"endTime\": \"2026-08-25T11:21:35.927411954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4a700\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-4kwl7rkmbbsz\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55b4601f-fd16-4b14-83df-ce42375746d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55b4601f-fd16-4b14-83df-ce42375746d5\",\n \"name\": \"55b4601f-fd16-4b14-83df-ce42375746d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.796667754Z\",\n \"endTime\": \"2026-08-25T11:21:35.927411954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6486bc7f6e323c672fc95824dca88e99d53b68f78c7b8ca764eefba2665ef7d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"806a6cee782cfb1e97e02367c95a679ce5b8a00edfd0c58d876dd15273163cad","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000622d60\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-77gllg\" in resourcegroup=\"rg-np-downgrade-77gllg-4r7rgqsrgh66\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e1ce5887-74ea-4460-be42-e4173a17bfbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e1ce5887-74ea-4460-be42-e4173a17bfbe\",\n \"name\": \"e1ce5887-74ea-4460-be42-e4173a17bfbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:15:36.167245148Z\",\n \"endTime\": \"2026-08-25T11:34:45.86895076Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000622d60\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-77gllg\" in resourcegroup=\"rg-np-downgrade-77gllg-4r7rgqsrgh66\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e1ce5887-74ea-4460-be42-e4173a17bfbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e1ce5887-74ea-4460-be42-e4173a17bfbe\",\n \"name\": \"e1ce5887-74ea-4460-be42-e4173a17bfbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:15:36.167245148Z\",\n \"endTime\": \"2026-08-25T11:34:45.86895076Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ae90507e2cae94101df561cb0f76aee174beba0d6db42fe1f926fc15cd40b95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"732bac21dc6f40ccf236a6b2bee56772eb8544e2c5358966c3a2a55ba78a23ba","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-f8plrr at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011dc620\u003e: \n failed to create HCP cluster np-skip-minor-f8plrr: failed waiting for cluster=\"np-skip-minor-f8plrr\" in resourcegroup=\"rg-np-skip-minor-f8plrr-kgf8vn4q6tps\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bf4fc95-3caf-453e-9414-14bf5c581205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bf4fc95-3caf-453e-9414-14bf5c581205\",\n \"name\": \"6bf4fc95-3caf-453e-9414-14bf5c581205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.549945224Z\",\n \"endTime\": \"2026-08-25T11:21:35.854676439Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-f8plrr at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011dc620\u003e: \n failed to create HCP cluster np-skip-minor-f8plrr: failed waiting for cluster=\"np-skip-minor-f8plrr\" in resourcegroup=\"rg-np-skip-minor-f8plrr-kgf8vn4q6tps\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bf4fc95-3caf-453e-9414-14bf5c581205\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bf4fc95-3caf-453e-9414-14bf5c581205\",\n \"name\": \"6bf4fc95-3caf-453e-9414-14bf5c581205\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.549945224Z\",\n \"endTime\": \"2026-08-25T11:21:35.854676439Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, cluster-policy-controller, cluster-storage-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e23ad42416ed5da4b097999f78974532f78fb6bae4721159e3692b0e00302eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"46cface1afceb818229ce74688d353ed8c60a22a002f15862ba902e03ea766aa","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff61c0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-d6scg6nrwppk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bb3d540-0c63-47be-ae4b-406cef03fed4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bb3d540-0c63-47be-ae4b-406cef03fed4\",\n \"name\": \"7bb3d540-0c63-47be-ae4b-406cef03fed4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.420307394Z\",\n \"endTime\": \"2026-08-25T11:21:36.057718306Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff61c0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-d6scg6nrwppk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bb3d540-0c63-47be-ae4b-406cef03fed4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bb3d540-0c63-47be-ae4b-406cef03fed4\",\n \"name\": \"7bb3d540-0c63-47be-ae4b-406cef03fed4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.420307394Z\",\n \"endTime\": \"2026-08-25T11:21:36.057718306Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d90d92160584a35c29b41f61dd81e65e8420705e7c4f675152d1847a425ac05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"a8c7c70bd3d6dd8630d347b4d24584ab9920ea0b8c33dde6ba59740ac0218935","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002180\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-qxgjhv2nzd4n\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359c8bcf-a5fe-4f3a-b5e8-9eeaeaf92731\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359c8bcf-a5fe-4f3a-b5e8-9eeaeaf92731\",\n \"name\": \"359c8bcf-a5fe-4f3a-b5e8-9eeaeaf92731\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.8394171Z\",\n \"endTime\": \"2026-08-25T11:21:36.044301402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, openshift-route-controller-manager, packageserver, csi-snapshot-controller-operator, cluster-version-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002180\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-qxgjhv2nzd4n\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359c8bcf-a5fe-4f3a-b5e8-9eeaeaf92731\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359c8bcf-a5fe-4f3a-b5e8-9eeaeaf92731\",\n \"name\": \"359c8bcf-a5fe-4f3a-b5e8-9eeaeaf92731\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.8394171Z\",\n \"endTime\": \"2026-08-25T11:21:36.044301402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, olm-operator, openshift-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, openshift-route-controller-manager, packageserver, csi-snapshot-controller-operator, cluster-version-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ffd7abe6880dad2f185788d005cbc4bd7dae2d19fc8c7cbbf60ebcfe13e7cdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"bc8afc40785a7de130f0afb33506490a59b6681bd8c5fec932af8efd346aaf8c","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e1a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nlgtzd8jj8ls\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"name\": \"5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.862453094Z\",\n \"endTime\": \"2026-08-25T11:21:35.935998302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e1a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nlgtzd8jj8ls\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"name\": \"5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.862453094Z\",\n \"endTime\": \"2026-08-25T11:21:35.935998302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b460f0e72ca79f88845e6e8bf228b8d0be2539d0f8bce034dfbee5009ef88258","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"909ea8456e14e77e7c695a66fe70572fbb700e8d132116d69fc1b6f11556b6fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"a3f04b099b92e17296b69b5c0846c2aba1e01c878468735dffc3852678c7ee27","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-k9tccs with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c61e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-k9tccs: failed waiting for cluster=\"np-version-upgrade-cluster-k9tccs\" in resourcegroup=\"rg-np-version-upgrade-k9tccs-dxkthfx2xcmq\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"name\": \"06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.260205158Z\",\n \"endTime\": \"2026-08-25T11:21:35.932791726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-k9tccs with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c61e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-k9tccs: failed waiting for cluster=\"np-version-upgrade-cluster-k9tccs\" in resourcegroup=\"rg-np-version-upgrade-k9tccs-dxkthfx2xcmq\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"name\": \"06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:30.260205158Z\",\n \"endTime\": \"2026-08-25T11:21:35.932791726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"96f2ef48dc9df65d1aee7cc3bd73c0b2a7f80d3a10face1d40b82bad42378bba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"429e22ec2d21dbf61e0c7b6393566a722dc292f14b64db5f2803331be42369e0","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7e7a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t47tntcsxrpm\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afe963d2-3618-4199-8c62-b39caa628b25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afe963d2-3618-4199-8c62-b39caa628b25\",\n \"name\": \"afe963d2-3618-4199-8c62-b39caa628b25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.270419413Z\",\n \"endTime\": \"2026-08-25T11:21:36.047784787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-version-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, olm-operator, cluster-network-operator, catalog-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7e7a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t47tntcsxrpm\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afe963d2-3618-4199-8c62-b39caa628b25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afe963d2-3618-4199-8c62-b39caa628b25\",\n \"name\": \"afe963d2-3618-4199-8c62-b39caa628b25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.270419413Z\",\n \"endTime\": \"2026-08-25T11:21:36.047784787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-version-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, olm-operator, cluster-network-operator, catalog-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9fde85fdc5d954dded681bba1df0d5a0ce6c39f48eda2821524084debc03f8f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"e075d605863206fdc2ccadedfa2c3d789e15cebb06d5f64e9eeed7ecaf6fde31","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-vk6cfm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001216240\u003e: \n failed to create HCP cluster np-noedge-vk6cfm: failed waiting for cluster=\"np-noedge-vk6cfm\" in resourcegroup=\"rg-np-noedge-vk6cfm-r5z9ggrs4bzc\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"name\": \"6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:40.855489688Z\",\n \"endTime\": \"2026-08-25T11:21:45.894005118Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, packageserver, olm-collect-profiles, community-operators-catalog, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, cluster-autoscaler, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, openshift-apiserver, cluster-image-registry-operator, certified-operators-catalog, capi-provider, openshift-route-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, ignition-server, olm-operator, machine-approver, kube-controller-manager, kube-scheduler, cluster-network-operator, control-plane-pki-operator, openshift-controller-manager, redhat-operators-catalog, cluster-node-tuning-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-vk6cfm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001216240\u003e: \n failed to create HCP cluster np-noedge-vk6cfm: failed waiting for cluster=\"np-noedge-vk6cfm\" in resourcegroup=\"rg-np-noedge-vk6cfm-r5z9ggrs4bzc\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"name\": \"6d73055c-caa7-4581-b662-b31a5154ad44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:40.855489688Z\",\n \"endTime\": \"2026-08-25T11:21:45.894005118Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, packageserver, olm-collect-profiles, community-operators-catalog, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, cluster-autoscaler, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, openshift-apiserver, cluster-image-registry-operator, certified-operators-catalog, capi-provider, openshift-route-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, ignition-server, olm-operator, machine-approver, kube-controller-manager, kube-scheduler, cluster-network-operator, control-plane-pki-operator, openshift-controller-manager, redhat-operators-catalog, cluster-node-tuning-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de9ee73e1f42212ffe8bacf6081359c2c391b2c47c68ba07fa4cd11f00e7b015","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a4ccdf841925d858dbdf4d8f6fc6f52c8cbabe823dfde81ca5dc81039cb77bcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"b6335e195fbae35825fcde7af066eb7de6c819b9fcc823ca2c6c99c2440ef185","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d66520\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-k4pms9xqtc7k\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d5ee1a7-b048-4225-ae5f-fd060820ad04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d5ee1a7-b048-4225-ae5f-fd060820ad04\",\n \"name\": \"5d5ee1a7-b048-4225-ae5f-fd060820ad04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.062224891Z\",\n \"endTime\": \"2026-08-25T11:21:35.945858091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, openshift-apiserver, community-operators-catalog, cluster-policy-controller, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, cluster-image-registry-operator, olm-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d66520\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-k4pms9xqtc7k\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d5ee1a7-b048-4225-ae5f-fd060820ad04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d5ee1a7-b048-4225-ae5f-fd060820ad04\",\n \"name\": \"5d5ee1a7-b048-4225-ae5f-fd060820ad04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.062224891Z\",\n \"endTime\": \"2026-08-25T11:21:35.945858091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, ingress-operator, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, openshift-apiserver, community-operators-catalog, cluster-policy-controller, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, cluster-image-registry-operator, olm-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9f523e967732c1b3d07dee37f3e82fa3c4ce87cdabd49b89d1d6af9040db8e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"9c1b8b2a2c406c20c5278d4329b8f1b42ee01e05e65a61c633a54cbe6a5372e5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-l5fd4zcnxvvv/cluster-candidate-4-20-p4jhqx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001798120\u003e: \n failed to create HCP cluster cluster-candidate-4-20-p4jhqx: failed waiting for cluster=\"cluster-candidate-4-20-p4jhqx\" in resourcegroup=\"rg-candidate-4-20-l5fd4zcnxvvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fee2995-90ed-423d-883d-12c988a9efaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fee2995-90ed-423d-883d-12c988a9efaa\",\n \"name\": \"0fee2995-90ed-423d-883d-12c988a9efaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.883200116Z\",\n \"endTime\": \"2026-08-25T11:21:35.924623167Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, ingress-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, redhat-marketplace-catalog, kube-controller-manager, azure-cloud-controller-manager, cluster-network-operator, dns-operator, ignition-server, olm-operator, packageserver, cluster-version-operator, ignition-server-proxy, olm-collect-profiles, cluster-autoscaler, konnectivity-agent, control-plane-pki-operator, openshift-apiserver, redhat-operators-catalog, machine-approver, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-image-registry-operator, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-l5fd4zcnxvvv/cluster-candidate-4-20-p4jhqx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001798120\u003e: \n failed to create HCP cluster cluster-candidate-4-20-p4jhqx: failed waiting for cluster=\"cluster-candidate-4-20-p4jhqx\" in resourcegroup=\"rg-candidate-4-20-l5fd4zcnxvvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fee2995-90ed-423d-883d-12c988a9efaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fee2995-90ed-423d-883d-12c988a9efaa\",\n \"name\": \"0fee2995-90ed-423d-883d-12c988a9efaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.883200116Z\",\n \"endTime\": \"2026-08-25T11:21:35.924623167Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, ingress-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, redhat-marketplace-catalog, kube-controller-manager, azure-cloud-controller-manager, cluster-network-operator, dns-operator, ignition-server, olm-operator, packageserver, cluster-version-operator, ignition-server-proxy, olm-collect-profiles, cluster-autoscaler, konnectivity-agent, control-plane-pki-operator, openshift-apiserver, redhat-operators-catalog, machine-approver, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-image-registry-operator, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad5493912f43bcb9ccc55c0823878ed2c5dac10d9b3998d7eae815c734f5a972","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"6e634d35cd843e9cfae5b6b27514de0cb7373b65ef36dac104cfb1c90125fcc9","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-vmkxjb\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001438440\u003e: \n failed to create HCP cluster cluster-zstream-4-22-vmkxjb: failed waiting for cluster=\"cluster-zstream-4-22-vmkxjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-dgzc9xl48h57\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\",\n \"name\": \"88ef8722-4c84-4313-a177-16c92648110e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:49.211451249Z\",\n \"endTime\": \"2026-08-25T11:21:55.960423286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-vmkxjb\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001438440\u003e: \n failed to create HCP cluster cluster-zstream-4-22-vmkxjb: failed waiting for cluster=\"cluster-zstream-4-22-vmkxjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-dgzc9xl48h57\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\",\n \"name\": \"88ef8722-4c84-4313-a177-16c92648110e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:49.211451249Z\",\n \"endTime\": \"2026-08-25T11:21:55.960423286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"60a907245af237f1fcb597ff0d35bc01af7bd1e3900650496d648ae7c2abbb63","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b01fd47119eccea8b44ad519f1e144c83c5d78b1429db8804af1b98c447ba796","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"f070923f4a7c1696bb4766ac883978b74411c5db4f343ca2e398a5b72fdf0f61","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zdj2mslnj72d/cluster-candidate-5-0-km49gg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0c4a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-km49gg: failed waiting for cluster=\"cluster-candidate-5-0-km49gg\" in resourcegroup=\"rg-candidate-5-0-zdj2mslnj72d\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecb18d64-1259-4b7c-8758-a10a364e1ea8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecb18d64-1259-4b7c-8758-a10a364e1ea8\",\n \"name\": \"ecb18d64-1259-4b7c-8758-a10a364e1ea8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.10152748Z\",\n \"endTime\": \"2026-08-25T11:21:35.97429848Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, openshift-controller-manager, cluster-storage-operator, ingress-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-network-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, catalog-operator, redhat-operators-catalog, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zdj2mslnj72d/cluster-candidate-5-0-km49gg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0c4a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-km49gg: failed waiting for cluster=\"cluster-candidate-5-0-km49gg\" in resourcegroup=\"rg-candidate-5-0-zdj2mslnj72d\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecb18d64-1259-4b7c-8758-a10a364e1ea8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecb18d64-1259-4b7c-8758-a10a364e1ea8\",\n \"name\": \"ecb18d64-1259-4b7c-8758-a10a364e1ea8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.10152748Z\",\n \"endTime\": \"2026-08-25T11:21:35.97429848Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, openshift-controller-manager, cluster-storage-operator, ingress-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-network-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, catalog-operator, redhat-operators-catalog, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b7488ff419a87f6765512deaa82bd09608faeea9da633c2c63abd2726bc8415e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"37180a3567b7ba4c19f170d6ea685b79ea6d88f1f355e96431dbe0380eee9693","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a82300\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-jzcdxxl8w9bb\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f907cfb0-07c7-4e5e-a54f-7c85f0924872\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f907cfb0-07c7-4e5e-a54f-7c85f0924872\",\n \"name\": \"f907cfb0-07c7-4e5e-a54f-7c85f0924872\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:39.639932862Z\",\n \"endTime\": \"2026-08-25T11:21:45.83743101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a82300\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-jzcdxxl8w9bb\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f907cfb0-07c7-4e5e-a54f-7c85f0924872\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f907cfb0-07c7-4e5e-a54f-7c85f0924872\",\n \"name\": \"f907cfb0-07c7-4e5e-a54f-7c85f0924872\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:39.639932862Z\",\n \"endTime\": \"2026-08-25T11:21:45.83743101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8019368914e4f7c422059246ac58af27eeefebcf66a2043d13eff2a23cfbfe0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"a5d387bcc562b88ab6a23d38e6d2aa3de0eb793a989249e8ad448238a04782c1","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3cae0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jfmt6ghkl4r6\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"name\": \"276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.221378186Z\",\n \"endTime\": \"2026-08-25T11:21:35.933589807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3cae0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jfmt6ghkl4r6\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"name\": \"276ea232-8f07-40f6-a841-fed05291e9a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.221378186Z\",\n \"endTime\": \"2026-08-25T11:21:35.933589807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"60a907245af237f1fcb597ff0d35bc01af7bd1e3900650496d648ae7c2abbb63","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b9f1e6b747b678ed812c0a83454b5693ad22602d519240891fb5acb2b87fb781","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"59617915775eae13a40addb69d6b99b7fe7f688ae59ce79fcd656898578c9492","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048a80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-znbww55lj6vk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\",\n \"name\": \"32f30388-3946-451c-abc4-89ec972b44f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:32.564773496Z\",\n \"endTime\": \"2026-08-25T11:21:36.023308749Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, openshift-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048a80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-znbww55lj6vk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\",\n \"name\": \"32f30388-3946-451c-abc4-89ec972b44f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:32.564773496Z\",\n \"endTime\": \"2026-08-25T11:21:36.023308749Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, openshift-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fc2f22952e29415c0ffa879b795fd94133860d84ce001832877fc2b42797fd71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb9e38a8fb0f5cbc73de403a2848468416ad9d9b0a9adb25527cc94a6c3548fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"c225c2a52910bf841975f6f1ec954208c38deddccb41ad8c342c3c1da418ff75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"17edb3dd260ea059c244167666ed36031cba43f5f67e78698ae6697d0e919add","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001440fe0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-j7x6hnwtqnqh\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0510262-5da5-43b8-8474-f2a1f8d64bb5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0510262-5da5-43b8-8474-f2a1f8d64bb5\",\n \"name\": \"f0510262-5da5-43b8-8474-f2a1f8d64bb5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:11.358657089Z\",\n \"endTime\": \"2026-08-25T11:22:15.937634573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001440fe0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-j7x6hnwtqnqh\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0510262-5da5-43b8-8474-f2a1f8d64bb5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0510262-5da5-43b8-8474-f2a1f8d64bb5\",\n \"name\": \"f0510262-5da5-43b8-8474-f2a1f8d64bb5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:11.358657089Z\",\n \"endTime\": \"2026-08-25T11:22:15.937634573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c2e2ebbc62acada501f782d6e9cda81d34b8179272e05faed618fa6c80a09d0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"9021161863b438a4babb0dca655669309db8e7ffa0c5f505c949b37f07510077","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnnzd8dv8sc2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"name\": \"d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:53.022759327Z\",\n \"endTime\": \"2026-08-25T11:22:55.91053294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnnzd8dv8sc2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"name\": \"d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:53.022759327Z\",\n \"endTime\": \"2026-08-25T11:22:55.91053294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cf68c98c025edd528c92dbce3f73aec8e93c23a7a13306fd1115003353b267f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"08d1216f17f57d3c84a42ade2f863024d6549f84bec76f38daa65fb0c0f271e4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad6ee0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vb2vq2d7pgb7\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\",\n \"name\": \"9e043488-17a2-4487-83fe-b621f2697284\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:38.82687874Z\",\n \"endTime\": \"2026-08-25T11:22:45.901459654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, dns-operator, cluster-network-operator, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad6ee0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vb2vq2d7pgb7\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\",\n \"name\": \"9e043488-17a2-4487-83fe-b621f2697284\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:38.82687874Z\",\n \"endTime\": \"2026-08-25T11:22:45.901459654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, dns-operator, cluster-network-operator, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f135b58213daed6ea6fbc2013a314b004e48b63448926884614927be5a42fc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d4a447ffd3cb36d94863aca6933e1d2165f48cd6fd062c8c64784896260eefcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"2b601911adad56c98c50dd26e4c497713a18ded9cd009a058cdef7ea37a3ab49","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a4120\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2gtwfxshsqrl\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b7e1f60-4e89-4cc6-9b44-8aa6ef69312f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b7e1f60-4e89-4cc6-9b44-8aa6ef69312f\",\n \"name\": \"0b7e1f60-4e89-4cc6-9b44-8aa6ef69312f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.28378743Z\",\n \"endTime\": \"2026-08-25T11:21:35.930109254Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a4120\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2gtwfxshsqrl\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b7e1f60-4e89-4cc6-9b44-8aa6ef69312f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b7e1f60-4e89-4cc6-9b44-8aa6ef69312f\",\n \"name\": \"0b7e1f60-4e89-4cc6-9b44-8aa6ef69312f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.28378743Z\",\n \"endTime\": \"2026-08-25T11:21:35.930109254Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d7ad95ab0992a9d082ed7adf741007288fa08068a9022f72687500039ba135bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"ae1dbdff3202605f5db0ab0f7e502b028b65320bf6e7bd52d1ff183a02717a24","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-w6h87f\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001936140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-w6h87f: failed waiting for cluster=\"cluster-zstream-4-20-w6h87f\" in resourcegroup=\"rg-zstream-upgrade-4-20-xmx8p29fpbx4\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\",\n \"name\": \"a563416d-3979-490a-9710-b74e5cae9107\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.203167213Z\",\n \"endTime\": \"2026-08-25T11:21:35.888478672Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-w6h87f\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001936140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-w6h87f: failed waiting for cluster=\"cluster-zstream-4-20-w6h87f\" in resourcegroup=\"rg-zstream-upgrade-4-20-xmx8p29fpbx4\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\",\n \"name\": \"a563416d-3979-490a-9710-b74e5cae9107\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.203167213Z\",\n \"endTime\": \"2026-08-25T11:21:35.888478672Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"60a907245af237f1fcb597ff0d35bc01af7bd1e3900650496d648ae7c2abbb63","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2cb2f5f2aa8cd812d6e685a6511e12d8784b40605eed54e0a194d5b270127d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"5c82c50cdd5f477dc9bec1b74ec5428943202baab54c500d7530303ea24f62fe","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008526a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8svbm7w8gvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\",\n \"name\": \"37ed00b3-c155-4497-b144-33a7a3897320\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.292639561Z\",\n \"endTime\": \"2026-08-25T11:21:35.949263958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008526a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8svbm7w8gvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\",\n \"name\": \"37ed00b3-c155-4497-b144-33a7a3897320\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.292639561Z\",\n \"endTime\": \"2026-08-25T11:21:35.949263958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ef607a572d4c53f3a2663eb448d653e968a80b7e9c4d5dd17a7cbefc30c64148","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f0caa9f93f4efaff52c82090d36d5c907ede23fffd840f949196fe4b00f853d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"ab0efe968c3af5d317e92a58ea9e41044d2452ac55e715d3c4270447bf41e742","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000d9b260\u003e: \n GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85effd97-ba13-47a3-885a-a4eaa4cf5abc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85effd97-ba13-47a3-885a-a4eaa4cf5abc\",\n \"name\": \"85effd97-ba13-47a3-885a-a4eaa4cf5abc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:32.916616484Z\",\n \"endTime\": \"2026-08-25T11:22:35.913678576Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000d9b260\u003e: \n GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85effd97-ba13-47a3-885a-a4eaa4cf5abc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85effd97-ba13-47a3-885a-a4eaa4cf5abc\",\n \"name\": \"85effd97-ba13-47a3-885a-a4eaa4cf5abc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:32.916616484Z\",\n \"endTime\": \"2026-08-25T11:22:35.913678576Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7ab13fb655e17f3d62e310f2f8cf1fdf53f4f8c5ba0cc042157ad6bcb9ade7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","failed_at":"e2e","signature_id":"fb329634cdbcd47b225aa1460092969d8489ffd2ea14b20506198098253816f6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-bxvq84\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00480\u003e: \n failed to create HCP cluster cluster-zstream-4-21-bxvq84: failed waiting for cluster=\"cluster-zstream-4-21-bxvq84\" in resourcegroup=\"rg-zstream-upgrade-4-21-f772hgcp6g6p\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"name\": \"c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.100638601Z\",\n \"endTime\": \"2026-08-25T11:21:36.009069993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-image-registry-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, machine-approver, azure-cloud-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, hosted-cluster-config-operator, dns-operator, cluster-policy-controller, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, openshift-route-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-bxvq84\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00480\u003e: \n failed to create HCP cluster cluster-zstream-4-21-bxvq84: failed waiting for cluster=\"cluster-zstream-4-21-bxvq84\" in resourcegroup=\"rg-zstream-upgrade-4-21-f772hgcp6g6p\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"name\": \"c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:29.100638601Z\",\n \"endTime\": \"2026-08-25T11:21:36.009069993Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-image-registry-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, machine-approver, azure-cloud-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, hosted-cluster-config-operator, dns-operator, cluster-policy-controller, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, openshift-route-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6200cb8cea81a44cbffdcbf30f49e1b2e8791793e7e07d52640fca8ceb6c92c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":40,"failure_pattern_ids":["1264ce1f65fb1c4e4b03ef0f0b493117b1b9becb99a5e03d914e5ae33ea84790","1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","60a907245af237f1fcb597ff0d35bc01af7bd1e3900650496d648ae7c2abbb63","6200cb8cea81a44cbffdcbf30f49e1b2e8791793e7e07d52640fca8ceb6c92c0","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","b460f0e72ca79f88845e6e8bf228b8d0be2539d0f8bce034dfbee5009ef88258","b67b2954ed80e23d5dbe844cbe69ddf2aee82b77fbb4d9b923a377beae494cff","de9ee73e1f42212ffe8bacf6081359c2c391b2c47c68ba07fa4cd11f00e7b015","e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","ef607a572d4c53f3a2663eb448d653e968a80b7e9c4d5dd17a7cbefc30c64148","f135b58213daed6ea6fbc2013a314b004e48b63448926884614927be5a42fc0d","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","fc2f22952e29415c0ffa879b795fd94133860d84ce001832877fc2b42797fd71"],"matched_occurrences":40,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"d5a9f482aee0f501c8ae8bda8cd06ffa03214aaf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:02:51Z"},"failed_at":["alert","e2e"],"failed_test_count":22,"occurrences":[{"row_id":"002d1c17b7a067d5e7c0b390b3202e17187ded022cf24e39319ff1522a14f28e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"a557455b2f15d5267c35100305fbb366e3ce937bce05ea40aa0c0c92aaacdae2","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e24460\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cmpnvt9r42hk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"name\": \"f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:09.116151219Z\",\n \"endTime\": \"2026-08-25T11:17:17.145229508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e24460\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cmpnvt9r42hk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"name\": \"f79a299e-0c13-4f75-b94f-813c143c5081\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:09.116151219Z\",\n \"endTime\": \"2026-08-25T11:17:17.145229508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5f2092ac8b60125a2e24e70258e522230cf6619c6ea4fc2f4a10c6a15fc6e2e8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0114e1aa16ca99e4ee00ea685292a4e138aafa635c0f752065f2aa8decbbb1b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"b5ee1f8ad40d4305b4dfae8b6e148d13794c07d1d9f1c5fc72025bff8e810890","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d6c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-sbv7dmbdc9np\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d470f9f0-c744-4b72-8591-daf0fd43c83c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d470f9f0-c744-4b72-8591-daf0fd43c83c\",\n \"name\": \"d470f9f0-c744-4b72-8591-daf0fd43c83c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:19.10881324Z\",\n \"endTime\": \"2026-08-25T11:17:27.223455417Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d6c0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-sbv7dmbdc9np\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d470f9f0-c744-4b72-8591-daf0fd43c83c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d470f9f0-c744-4b72-8591-daf0fd43c83c\",\n \"name\": \"d470f9f0-c744-4b72-8591-daf0fd43c83c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:19.10881324Z\",\n \"endTime\": \"2026-08-25T11:17:27.223455417Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1148ac1b9c05c06e192b3e799be4946b60a4fa38fd993e96231ab944237c0586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"87423fb812ad03870c503443bdcecefa13a470d2f72adb08656f725ea5a893fd","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001008800\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gx8j8gn42w5n\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b44b8a6-930b-4456-b311-c56ef9cad9cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b44b8a6-930b-4456-b311-c56ef9cad9cc\",\n \"name\": \"4b44b8a6-930b-4456-b311-c56ef9cad9cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:57:58.872615618Z\",\n \"endTime\": \"2026-08-25T11:17:07.212466024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001008800\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gx8j8gn42w5n\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b44b8a6-930b-4456-b311-c56ef9cad9cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b44b8a6-930b-4456-b311-c56ef9cad9cc\",\n \"name\": \"4b44b8a6-930b-4456-b311-c56ef9cad9cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:57:58.872615618Z\",\n \"endTime\": \"2026-08-25T11:17:07.212466024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"165d18451faa409fd4ad8ac44883f2d4f8b08b6fa4bab382c0f8551eb8c1d3bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"9862cf1db12a585c6af07719f78d5cb393fa5c03f36fe7693b11f8ee1e06cab2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f19b00\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-d9n6fcrmls97\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e25d7c1-e055-43eb-b802-965b2457d6f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e25d7c1-e055-43eb-b802-965b2457d6f2\",\n \"name\": \"8e25d7c1-e055-43eb-b802-965b2457d6f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:12:07.690976253Z\",\n \"endTime\": \"2026-08-25T11:31:09.260959759Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f19b00\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-d9n6fcrmls97\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e25d7c1-e055-43eb-b802-965b2457d6f2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e25d7c1-e055-43eb-b802-965b2457d6f2\",\n \"name\": \"8e25d7c1-e055-43eb-b802-965b2457d6f2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:12:07.690976253Z\",\n \"endTime\": \"2026-08-25T11:31:09.260959759Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1e9ca4b844f78d74e72534ad161fe6f2cb2d8b808fb99c2cf548a15291e146a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"f8dc4e77835e39b0b6cc3a84dbc11fd7f2cbf76af6acd98523b7cdc62c8c4941","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:127]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eebc20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sbzff66jbsrl\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0db99600-a652-496e-bb1d-7e001a578b70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0db99600-a652-496e-bb1d-7e001a578b70\",\n \"name\": \"0db99600-a652-496e-bb1d-7e001a578b70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:10:45.087289014Z\",\n \"endTime\": \"2026-08-25T11:29:49.180304907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:127]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eebc20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sbzff66jbsrl\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0db99600-a652-496e-bb1d-7e001a578b70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0db99600-a652-496e-bb1d-7e001a578b70\",\n \"name\": \"0db99600-a652-496e-bb1d-7e001a578b70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:10:45.087289014Z\",\n \"endTime\": \"2026-08-25T11:29:49.180304907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"24730eed02802af4aec8a6bf892d53c33aac456701eafe8fc626524fb8965d11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"2481cd9dd55cb11b365e311cdeb7a7671bbaebdfc0f19a6ea5043f7d815bc754","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-kj48984pw47r/cluster-candidate-5-0-hjtsmj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ea2c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-hjtsmj: failed waiting for cluster=\"cluster-candidate-5-0-hjtsmj\" in resourcegroup=\"rg-candidate-5-0-kj48984pw47r\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72a51abb-1bf5-4be2-960e-dabd14ac1d8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72a51abb-1bf5-4be2-960e-dabd14ac1d8d\",\n \"name\": \"72a51abb-1bf5-4be2-960e-dabd14ac1d8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:08.811547737Z\",\n \"endTime\": \"2026-08-25T11:17:17.224243602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-kj48984pw47r/cluster-candidate-5-0-hjtsmj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ea2c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-hjtsmj: failed waiting for cluster=\"cluster-candidate-5-0-hjtsmj\" in resourcegroup=\"rg-candidate-5-0-kj48984pw47r\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72a51abb-1bf5-4be2-960e-dabd14ac1d8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/72a51abb-1bf5-4be2-960e-dabd14ac1d8d\",\n \"name\": \"72a51abb-1bf5-4be2-960e-dabd14ac1d8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:08.811547737Z\",\n \"endTime\": \"2026-08-25T11:17:17.224243602Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"26e047ae319a87b0c8eba91bb821cf0648680cff29ef0a1140358d0af8082fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"alert","signature_id":"1418018ee02cdd89dd6ac2c5c6e1378a62f42c5f58ff6ea19b95f13d3e928b4a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 35 time(s) (5 unknown, 30 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:11:38Z\n Ended: 2026-08-25T11:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8697984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdif0lntiqdeb940mhoffbkeopnpu9p-private-kas\", pod=\"router-645d46fdc5-79jdn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdif0lntiqdeb940mhoffbkeopnpu9p-private-kas/router-645d46fdc5-79jdn has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:12:39Z\n Ended: 2026-08-25T11:18:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8697984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdif0ma3m6d8mncl4cc474jr39gm8dl-c2x1i0r4y1n2p0o\", pod=\"cluster-network-operator-bf5f5bc79-tq8wx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdif0ma3m6d8mncl4cc474jr39gm8dl-c2x1i0r4y1n2p0o/cluster-network-operator-bf5f5bc79-tq8wx has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:12:39Z\n Ended: 2026-08-25T11:18:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8697984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdieiveill55dgo37f791mf7ehsfotd-sre-hcp-cluster\", pod=\"ingress-operator-795fd6dfdf-hg5b5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdieiveill55dgo37f791mf7ehsfotd-sre-hcp-cluster/ingress-operator-795fd6dfdf-hg5b5 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:12:39Z\n Ended: 2026-08-25T11:18:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8697984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdieiveill55dgo37f791mf7ehsfotd-sre-hcp-cluster\", pod=\"cluster-network-operator-6c9db9877f-dhrmw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdieiveill55dgo37f791mf7ehsfotd-sre-hcp-cluster/cluster-network-operator-6c9db9877f-dhrmw has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:27:40Z\n Ended: 2026-08-25T11:33:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8697984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiebjnqq7rr45qo1q02pjcjb34hngm-u1s3f2b9m4y4u9r\", pod=\"network-node-identity-cf5f9c5f6-755bc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiebjnqq7rr45qo1q02pjcjb34hngm-u1s3f2b9m4y4u9r/network-node-identity-cf5f9c5f6-755bc has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"28bd33c5d47314788ff03510f447f8a32ec4e3ef0e8f447b804a195c8474fe54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"9228c5a0348f43b2f29683a574e74aac88f34daeeda42ce48207d937f6f8a3c7","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb81a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cj9jxr6srqdd\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5fe6681-df8c-46f4-b88a-dcdd52ebe331\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5fe6681-df8c-46f4-b88a-dcdd52ebe331\",\n \"name\": \"a5fe6681-df8c-46f4-b88a-dcdd52ebe331\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:18.76965128Z\",\n \"endTime\": \"2026-08-25T11:17:27.213657973Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb81a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cj9jxr6srqdd\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5fe6681-df8c-46f4-b88a-dcdd52ebe331\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5fe6681-df8c-46f4-b88a-dcdd52ebe331\",\n \"name\": \"a5fe6681-df8c-46f4-b88a-dcdd52ebe331\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:18.76965128Z\",\n \"endTime\": \"2026-08-25T11:17:27.213657973Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"35cd94c0f44faef12421a6028f34c6ddb3a315671ffd3ba821620df58aab7a36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"5d116f9528929d7006556d8f84e31a0f89f4b8bd4db46d055fca3f0978d9a496","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86d60\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hbpc2f67phzw\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/555056e8-4d19-4e95-b4c8-bcf607cfce34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/555056e8-4d19-4e95-b4c8-bcf607cfce34\",\n \"name\": \"555056e8-4d19-4e95-b4c8-bcf607cfce34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:59:29.551776647Z\",\n \"endTime\": \"2026-08-25T11:18:37.175325569Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86d60\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hbpc2f67phzw\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/555056e8-4d19-4e95-b4c8-bcf607cfce34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/555056e8-4d19-4e95-b4c8-bcf607cfce34\",\n \"name\": \"555056e8-4d19-4e95-b4c8-bcf607cfce34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:59:29.551776647Z\",\n \"endTime\": \"2026-08-25T11:18:37.175325569Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"39828244bf949630989159918ab0eae32827a80db9d3144158e8a9e4206387f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"1cb8534c4d4dcf75a5e26bdcd0fd6bf39f4b3093ae4d3e32c0a5e2dd8fbdb573","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-75n442g7r7zh/cluster-candidate-4-20-t7lsbd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082120\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t7lsbd: failed waiting for cluster=\"cluster-candidate-4-20-t7lsbd\" in resourcegroup=\"rg-candidate-4-20-75n442g7r7zh\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f62eb2c8-ac1a-42b0-a45d-8db7519497d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f62eb2c8-ac1a-42b0-a45d-8db7519497d3\",\n \"name\": \"f62eb2c8-ac1a-42b0-a45d-8db7519497d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:19.459653729Z\",\n \"endTime\": \"2026-08-25T11:17:27.222849959Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, ingress-operator, catalog-operator, community-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, packageserver, olm-operator, redhat-marketplace-catalog, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-75n442g7r7zh/cluster-candidate-4-20-t7lsbd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082120\u003e: \n failed to create HCP cluster cluster-candidate-4-20-t7lsbd: failed waiting for cluster=\"cluster-candidate-4-20-t7lsbd\" in resourcegroup=\"rg-candidate-4-20-75n442g7r7zh\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f62eb2c8-ac1a-42b0-a45d-8db7519497d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f62eb2c8-ac1a-42b0-a45d-8db7519497d3\",\n \"name\": \"f62eb2c8-ac1a-42b0-a45d-8db7519497d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:19.459653729Z\",\n \"endTime\": \"2026-08-25T11:17:27.222849959Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, ingress-operator, catalog-operator, community-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, certified-operators-catalog, openshift-route-controller-manager, packageserver, olm-operator, redhat-marketplace-catalog, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"403d8fd579498ccde056fe71fe278049615270748e91304ac7f664c0a6603242","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"f43c2f2c050759d71b643af6fc429ef7cf204b9a1f2b922e8824120be8851f15","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tp6lxr8bfgv2\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"name\": \"863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:10:58.76533919Z\",\n \"endTime\": \"2026-08-25T11:29:59.215293237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tp6lxr8bfgv2\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"name\": \"863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:10:58.76533919Z\",\n \"endTime\": \"2026-08-25T11:29:59.215293237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}},{"row_id":"568656c767387cf2b39ffba7f7dcf92d83eadd037efb95f0c6b59ed23bb80267","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"b6676f8c8a5cec645c6a41c81541d0d21fdc763921f2f8f5e84133e0d2a0b697","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-zm76p2 at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0a80\u003e: \n failed to create HCP cluster np-skip-minor-zm76p2: failed waiting for cluster=\"np-skip-minor-zm76p2\" in resourcegroup=\"rg-np-skip-minor-zm76p2-zgjql8zj9727\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50269510-0da8-4161-a819-8e472daa7784\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50269510-0da8-4161-a819-8e472daa7784\",\n \"name\": \"50269510-0da8-4161-a819-8e472daa7784\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:10.700652772Z\",\n \"endTime\": \"2026-08-25T11:17:17.240136518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-zm76p2 at version 4.22.11\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0a80\u003e: \n failed to create HCP cluster np-skip-minor-zm76p2: failed waiting for cluster=\"np-skip-minor-zm76p2\" in resourcegroup=\"rg-np-skip-minor-zm76p2-zgjql8zj9727\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50269510-0da8-4161-a819-8e472daa7784\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50269510-0da8-4161-a819-8e472daa7784\",\n \"name\": \"50269510-0da8-4161-a819-8e472daa7784\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:10.700652772Z\",\n \"endTime\": \"2026-08-25T11:17:17.240136518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70c090ccc2f980aabb507cba11b132232b19c71941264b96e08b05e27b86c1a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"84726c82ed97016f12844a47373fafc8ccb52c61c42bd82f0b0f6c4092b047bd","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:324]: failed to create worker nodepool for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105d8a0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gs86r2pvl8xk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7fa394b-6357-4697-a2fa-3981a5577b25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7fa394b-6357-4697-a2fa-3981a5577b25\",\n \"name\": \"f7fa394b-6357-4697-a2fa-3981a5577b25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:43.537740054Z\",\n \"endTime\": \"2026-08-25T11:25:47.394793135Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:324]: failed to create worker nodepool for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105d8a0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gs86r2pvl8xk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7fa394b-6357-4697-a2fa-3981a5577b25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7fa394b-6357-4697-a2fa-3981a5577b25\",\n \"name\": \"f7fa394b-6357-4697-a2fa-3981a5577b25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:06:43.537740054Z\",\n \"endTime\": \"2026-08-25T11:25:47.394793135Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"738184e1118d614199737ec2763b96b373ccd10acc37da5129f08fdb28c54b3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"16c6bbce9d00cd3fdb259dff38344220eb8bc09eaf6c7024a6b9039971482f59","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c79240\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-n6jswvrxrm5x\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ce3ed2e-b25e-4f6a-b009-44359158a0c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ce3ed2e-b25e-4f6a-b009-44359158a0c0\",\n \"name\": \"4ce3ed2e-b25e-4f6a-b009-44359158a0c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:59:12.482119209Z\",\n \"endTime\": \"2026-08-25T11:18:17.224952479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c79240\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-n6jswvrxrm5x\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ce3ed2e-b25e-4f6a-b009-44359158a0c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ce3ed2e-b25e-4f6a-b009-44359158a0c0\",\n \"name\": \"4ce3ed2e-b25e-4f6a-b009-44359158a0c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:59:12.482119209Z\",\n \"endTime\": \"2026-08-25T11:18:17.224952479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c997637b48aea32effa9eab4be18e782f23b85a2340496b7c2acc3024c51d59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"9d9e7c31a0eab79ec1990c199b21ca4f9e3b8abb4900261837125436f88781d9","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e2a1a0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kg9bjcw5qtdg\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28b59e86-f017-47aa-9051-f2a88c01f26d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28b59e86-f017-47aa-9051-f2a88c01f26d\",\n \"name\": \"28b59e86-f017-47aa-9051-f2a88c01f26d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:18.609372122Z\",\n \"endTime\": \"2026-08-25T11:17:27.223023215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e2a1a0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kg9bjcw5qtdg\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28b59e86-f017-47aa-9051-f2a88c01f26d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28b59e86-f017-47aa-9051-f2a88c01f26d\",\n \"name\": \"28b59e86-f017-47aa-9051-f2a88c01f26d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:18.609372122Z\",\n \"endTime\": \"2026-08-25T11:17:27.223023215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"995fd0da260e642930be9b962b7a2230c2cb33a3d38808d39cb75b5ab52f8648","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"74b264555e08507bc1b8cd31ab8e6916b262467b913f5056245709dac63bf473","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fwpq26\" for cluster \"cluster-candidate-4-22-fwpq26\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee0a00\u003e: \n failed to create NodePool np-fwpq26: failed waiting for nodepool=\"np-fwpq26\" for cluster \"cluster-candidate-4-22-fwpq26\" in resourcegroup=\"rg-candidate-4-22-vdf2b8d7n27r\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1cd59521-c6cb-4009-a07d-22f759f51191\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1cd59521-c6cb-4009-a07d-22f759f51191\",\n \"name\": \"1cd59521-c6cb-4009-a07d-22f759f51191\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:15:37.131944946Z\",\n \"endTime\": \"2026-08-25T11:34:39.183255279Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fwpq26\" for cluster \"cluster-candidate-4-22-fwpq26\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee0a00\u003e: \n failed to create NodePool np-fwpq26: failed waiting for nodepool=\"np-fwpq26\" for cluster \"cluster-candidate-4-22-fwpq26\" in resourcegroup=\"rg-candidate-4-22-vdf2b8d7n27r\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1cd59521-c6cb-4009-a07d-22f759f51191\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1cd59521-c6cb-4009-a07d-22f759f51191\",\n \"name\": \"1cd59521-c6cb-4009-a07d-22f759f51191\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:15:37.131944946Z\",\n \"endTime\": \"2026-08-25T11:34:39.183255279Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f237d065463221fe483cd34a7618758865972e4096c8bc0226839d3f8f95755","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"fd5838277e9b2418a0afbc2057c3ce5fc4bd346774696e4589e2c05f45f73f1e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-m6ppsj2p4fcx/cluster-candidate-4-21-twphbj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ba2e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-twphbj: failed waiting for cluster=\"cluster-candidate-4-21-twphbj\" in resourcegroup=\"rg-candidate-4-21-m6ppsj2p4fcx\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/677d8f28-f8a3-4aa6-9feb-e1997ff2d5a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/677d8f28-f8a3-4aa6-9feb-e1997ff2d5a8\",\n \"name\": \"677d8f28-f8a3-4aa6-9feb-e1997ff2d5a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:08.661213076Z\",\n \"endTime\": \"2026-08-25T11:17:17.099289477Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-m6ppsj2p4fcx/cluster-candidate-4-21-twphbj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ba2e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-twphbj: failed waiting for cluster=\"cluster-candidate-4-21-twphbj\" in resourcegroup=\"rg-candidate-4-21-m6ppsj2p4fcx\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/677d8f28-f8a3-4aa6-9feb-e1997ff2d5a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/677d8f28-f8a3-4aa6-9feb-e1997ff2d5a8\",\n \"name\": \"677d8f28-f8a3-4aa6-9feb-e1997ff2d5a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:58:08.661213076Z\",\n \"endTime\": \"2026-08-25T11:17:17.099289477Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b55e02572c2e99c43c4fa49fee75cfa75ccc3155389a5dc027d77f6a632a33b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"9f7d9da9375d115c3d27ec74ca79a5ddf3840aa4ba1cb9141a12954a39487c12","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c85320\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8h9ptxfrdc67\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"name\": \"0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:00:22.611774831Z\",\n \"endTime\": \"2026-08-25T11:19:27.266054327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, certified-operators-catalog, catalog-operator, olm-operator, openshift-apiserver, cluster-policy-controller, dns-operator, openshift-route-controller-manager, ignition-server-proxy, packageserver, redhat-operators-catalog, control-plane-pki-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c85320\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8h9ptxfrdc67\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"name\": \"0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:00:22.611774831Z\",\n \"endTime\": \"2026-08-25T11:19:27.266054327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, certified-operators-catalog, catalog-operator, olm-operator, openshift-apiserver, cluster-policy-controller, dns-operator, openshift-route-controller-manager, ignition-server-proxy, packageserver, redhat-operators-catalog, control-plane-pki-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6bc6e1fed29949cb7f899cfde9a499bb81c4498d10956c4f31ee3021f08a1b5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c36bf11a6b6254ee6701c38a38e38fd55c00d3c5222da196dc8a3c80f25524b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"e3ff43e82a0f930ac10e5802807ad5e255f69a88161d6322c6013bce16cd034d","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d060\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-2t5r9x2bsd4t\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c41ebf24-2fbc-424c-afac-9dcffe2ce8a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c41ebf24-2fbc-424c-afac-9dcffe2ce8a6\",\n \"name\": \"c41ebf24-2fbc-424c-afac-9dcffe2ce8a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:09:04.655676542Z\",\n \"endTime\": \"2026-08-25T11:28:09.149731834Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d060\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-2t5r9x2bsd4t\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c41ebf24-2fbc-424c-afac-9dcffe2ce8a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c41ebf24-2fbc-424c-afac-9dcffe2ce8a6\",\n \"name\": \"c41ebf24-2fbc-424c-afac-9dcffe2ce8a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:09:04.655676542Z\",\n \"endTime\": \"2026-08-25T11:28:09.149731834Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"da53e11ed9cff7620612a13f6b489e2f19d96ef27b564bd65b3595f672aa7e53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"e006fc8de127b51c0b9862eaa26e5d9a844309286eb194166ccd19b3c55c024d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008b0740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-52qjk2wv2ffd\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e88e29b0-abca-45a6-84b3-7185d4bb140b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e88e29b0-abca-45a6-84b3-7185d4bb140b\",\n \"name\": \"e88e29b0-abca-45a6-84b3-7185d4bb140b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:59:29.34918094Z\",\n \"endTime\": \"2026-08-25T11:18:37.197380239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008b0740\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-52qjk2wv2ffd\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e88e29b0-abca-45a6-84b3-7185d4bb140b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e88e29b0-abca-45a6-84b3-7185d4bb140b\",\n \"name\": \"e88e29b0-abca-45a6-84b3-7185d4bb140b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:59:29.34918094Z\",\n \"endTime\": \"2026-08-25T11:18:37.197380239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ddf4ff5b2caca47a4f428199736849015552cf27e8b9665a3773f781c7e6a196","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"alert","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:10:19Z\n Ended: 2026-08-25T11:46:21Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8697984-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"ecef48009ae8835a57ee6c4a4f5bf420fcfb9d18f5ef4f748173982fadd1056a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","failed_at":"e2e","signature_id":"1c7bb16e4b644044bda0b75b866f85ba0a96450750139ae786fdc9571148e984","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-b5mv5d\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000def1a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-b5mv5d: failed waiting for cluster=\"cluster-zstream-4-21-b5mv5d\" in resourcegroup=\"rg-zstream-upgrade-4-21-nfcd2w7bxk5w\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c78209-32c9-4764-a27b-84232443578b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c78209-32c9-4764-a27b-84232443578b\",\n \"name\": \"d2c78209-32c9-4764-a27b-84232443578b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:57:58.86149776Z\",\n \"endTime\": \"2026-08-25T11:17:07.258943288Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-b5mv5d\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000def1a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-b5mv5d: failed waiting for cluster=\"cluster-zstream-4-21-b5mv5d\" in resourcegroup=\"rg-zstream-upgrade-4-21-nfcd2w7bxk5w\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c78209-32c9-4764-a27b-84232443578b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c78209-32c9-4764-a27b-84232443578b\",\n \"name\": \"d2c78209-32c9-4764-a27b-84232443578b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:57:58.86149776Z\",\n \"endTime\": \"2026-08-25T11:17:07.258943288Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":22,"failure_pattern_ids":["03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","5f2092ac8b60125a2e24e70258e522230cf6619c6ea4fc2f4a10c6a15fc6e2e8","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","d6bc6e1fed29949cb7f899cfde9a499bb81c4498d10956c4f31ee3021f08a1b5","ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":22,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6638,"pr_state":"open","pr_sha":"c403afe8d857fa8c76e2da31fc1c9d3952ca11b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:02:45Z"},"failed_at":["alert","e2e"],"failed_test_count":48,"occurrences":[{"row_id":"017b13624931c40e24521810032711a0e0d03a7934648fca70ddc8be640b724c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"alert","signature_id":"7ed12c6bca5e7dc43cf2acfe3e266ecf9c33df65853c31ec85ae02d6ffec33ff","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:29:19Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors1h5m\", cluster=\"ci01-j5488384-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5488384/providers/microsoft.monitor/accounts/services-j5488384\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of cluster create (install) operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours. A regional install failure of this magnitude typically points at a shared dependency (e.g. registry, DNS, or ARM) rather than individual clusters.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470aab7d7a3605e8633a637a70d37da16684c6630946576af6d433f851290640","failure_pattern":"alert [svc] UJClusterProvisionErrors1h5m fired","search_query":"alert [svc] UJClusterProvisionErrors1h5m fired"}},{"row_id":"09fafff4fd60b82eba83a46145c90cab27e0646e18f2014bec2f8d8b260713c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"09f9bb566a2cac0401d2761a3e203049bfc5d3f036f5254838e3a2c316495d34","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd5080\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-d9wqh8hjrv2l\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba9a48b3-fad7-429f-8f89-6661e4793c74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba9a48b3-fad7-429f-8f89-6661e4793c74\",\n \"name\": \"ba9a48b3-fad7-429f-8f89-6661e4793c74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:23.188516579Z\",\n \"endTime\": \"2026-08-25T11:24:23.029260546Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd5080\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-d9wqh8hjrv2l\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba9a48b3-fad7-429f-8f89-6661e4793c74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba9a48b3-fad7-429f-8f89-6661e4793c74\",\n \"name\": \"ba9a48b3-fad7-429f-8f89-6661e4793c74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:23.188516579Z\",\n \"endTime\": \"2026-08-25T11:24:23.029260546Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b1042ede12834cc28ea8ecd19c54248414cf2e5a0c5859fc541509dcef4225b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"fe7dfc068c8c4065aa24dac07dd321a5e46b5ffb23c7bef9e08da4d256504475","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9ddgb764dmsn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"name\": \"7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.015923741Z\",\n \"endTime\": \"2026-08-25T11:23:43.160352144Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, packageserver, cluster-policy-controller, ingress-operator, olm-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9ddgb764dmsn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"name\": \"7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.015923741Z\",\n \"endTime\": \"2026-08-25T11:23:43.160352144Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, packageserver, cluster-policy-controller, ingress-operator, olm-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"deacd16fd42c584f7532a98cc9b198caf769401900c4d495ba1223ef893fffc3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0c632912820fb461e506d61bd40706ee7c30b8423fc10ecc7fcbc4debf539b15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"alert","signature_id":"e906e7ae8a41e3b71c67202b1c3d92e471ada05f8b75a59665632b01eb944b26","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 208 time(s) (81 unknown, 127 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfurlbg9kr3lc200gkudg3qrsi33-f5k4p3y8a6g9u0l\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfurlbg9kr3lc200gkudg3qrsi33-f5k4p3y8a6g9u0l/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhr5bq5su1ium6nlh4545uep5uje-basic-cluster\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhr5bq5su1ium6nlh4545uep5uje-basic-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:23:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjqmlu1hialupee3558f14mpu4fq-ea-cluster\", pod=\"ingress-operator-58b769b7cd-trz9l\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjqmlu1hialupee3558f14mpu4fq-ea-cluster/ingress-operator-58b769b7cd-trz9l has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihks11c1ah6qljd4mi7jgu0gu4n2v-ea-listfdq7jq\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihks11c1ah6qljd4mi7jgu0gu4n2v-ea-listfdq7jq/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:20:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihsu90nkfp7klvqiibrq8qignpkn3-g2n5z8y2z6s4g5l\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihsu90nkfp7klvqiibrq8qignpkn3-g2n5z8y2z6s4g5l/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjsm8auo5mc15flc0h8lullatcjv-t5i3m0e7c7g2q4m\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjsm8auo5mc15flc0h8lullatcjv-t5i3m0e7c7g2q4m/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:26:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjqmlu1hialupee3558f14mpu4fq-ea-cluster\", pod=\"cluster-network-operator-55979955d9-chts5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjqmlu1hialupee3558f14mpu4fq-ea-cluster/cluster-network-operator-55979955d9-chts5 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0rv1416jugfpm4c1sbi7ksv10ql-b1f1n2v7g9u5e7b\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0rv1416jugfpm4c1sbi7ksv10ql-b1f1n2v7g9u5e7b/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-25T11:14:44Z\n Ended: 2026-08-25T11:21:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjqmlu1hialupee3558f14mpu4fq-ea-cluster\", pod=\"cluster-policy-controller-b98bc7bb9-khp8v\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjqmlu1hialupee3558f14mpu4fq-ea-cluster/cluster-policy-controller-b98bc7bb9-khp8v has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-25T11:17:44Z\n Ended: 2026-08-25T11:24:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihcijeq1audlkvcatvpfdpj8kma3b-j9a9x3a9c6m6m9l\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihcijeq1audlkvcatvpfdpj8kma3b-j9a9x3a9c6m6m9l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-25T11:17:44Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihkgjeab02r8c0v0399qpv5f7jdpv-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihkgjeab02r8c0v0399qpv5f7jdpv-private-ingress/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-25T11:17:44Z\n Ended: 2026-08-25T11:26:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihv2u02bgedle3t00he6napcr1rvi-agg-cluster\", pod=\"router-86db9b6b78-2d4wm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihv2u02bgedle3t00he6napcr1rvi-agg-cluster/router-86db9b6b78-2d4wm has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-25T11:17:44Z\n Ended: 2026-08-25T11:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihcijeq1audlkvcatvpfdpj8kma3b-j9a9x3a9c6m6m9l\", pod=\"router-77bc884b98-2lq4f\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihcijeq1audlkvcatvpfdpj8kma3b-j9a9x3a9c6m6m9l/router-77bc884b98-2lq4f has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-25T11:17:44Z\n Ended: 2026-08-25T11:24:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihcijeq1audlkvcatvpfdpj8kma3b-j9a9x3a9c6m6m9l\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihcijeq1audlkvcatvpfdpj8kma3b-j9a9x3a9c6m6m9l/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c\", pod=\"csi-snapshot-controller-operator-5b78c5479b-vxvgz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c/csi-snapshot-controller-operator-5b78c5479b-vxvgz has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c\", pod=\"dns-operator-7d8cffff6-4qzgx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c/dns-operator-7d8cffff6-4qzgx has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-25T11:18:42Z\n Ended: 2026-08-25T11:24:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i\", pod=\"cluster-image-registry-operator-9cd45fd9c-rmjdj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i/cluster-image-registry-operator-9cd45fd9c-rmjdj has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-25T11:18:43Z\n Ended: 2026-08-25T11:24:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihftlhoa7up08rja7n6ld8cgfrvpa-e3s2m5z1z4m6v2v\", pod=\"openshift-apiserver-c6b8f956c-k45fh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihftlhoa7up08rja7n6ld8cgfrvpa-e3s2m5z1z4m6v2v/openshift-apiserver-c6b8f956c-k45fh has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-25T11:18:43Z\n Ended: 2026-08-25T11:27:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c\", pod=\"cluster-network-operator-65ccfd54c7-8x6vv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c/cluster-network-operator-65ccfd54c7-8x6vv has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-25T11:18:43Z\n Ended: 2026-08-25T11:24:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihftlhoa7up08rja7n6ld8cgfrvpa-e3s2m5z1z4m6v2v\", pod=\"ignition-server-proxy-5c4786958-7fw5g\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihftlhoa7up08rja7n6ld8cgfrvpa-e3s2m5z1z4m6v2v/ignition-server-proxy-5c4786958-7fw5g has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-25T11:18:44Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i\", pod=\"csi-snapshot-controller-operator-786784fd7b-vnl4v\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i/csi-snapshot-controller-operator-786784fd7b-vnl4v has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-25T11:18:44Z\n Ended: 2026-08-25T11:24:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c\", pod=\"cluster-storage-operator-78f86b5d97-r2zsk\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c/cluster-storage-operator-78f86b5d97-r2zsk has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-08-25T11:18:44Z\n Ended: 2026-08-25T11:28:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c\", pod=\"ingress-operator-6c97f54695-s68pf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c/ingress-operator-6c97f54695-s68pf has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-08-25T11:18:44Z\n Ended: 2026-08-25T11:24:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c\", pod=\"cluster-version-operator-67d94cd64d-kbh9b\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0u0tgeba2q4ut4gdbavl2edkv0e-n9p2o8n5f5n4v3c/cluster-version-operator-67d94cd64d-kbh9b has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-08-25T11:18:44Z\n Ended: 2026-08-25T11:24:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i\", pod=\"cluster-policy-controller-749c586f99-nwcsd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i/cluster-policy-controller-749c586f99-nwcsd has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:27:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p\", pod=\"cluster-network-operator-5d4f6bdfbb-8qzx6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p/cluster-network-operator-5d4f6bdfbb-8qzx6 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihknfg29t9kvecdq95udbscb5i26m-cilium-cluster\", pod=\"router-7fd5474f96-mgmpb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihknfg29t9kvecdq95udbscb5i26m-cilium-cluster/router-7fd5474f96-mgmpb has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:29:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p\", pod=\"dns-operator-ff7889f8d-5sphm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p/dns-operator-ff7889f8d-5sphm has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigme1iva31aldq1cv323josrfntfm-o9z2z2j3o7m8n4t\", pod=\"openshift-apiserver-6575b9fb56-9crm7\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigme1iva31aldq1cv323josrfntfm-o9z2z2j3o7m8n4t/openshift-apiserver-6575b9fb56-9crm7 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:26:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihv2u02bgedle3t00he6napcr1rvi-agg-cluster\", pod=\"kube-apiserver-55f4db9d5-jrvtz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihv2u02bgedle3t00he6napcr1rvi-agg-cluster/kube-apiserver-55f4db9d5-jrvtz has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-08-25T11:19:42Z\n Ended: 2026-08-25T11:25:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigov251107me410shcf9af2facoqc-oidc-wi-cluster\", pod=\"openshift-apiserver-6f6d6b88f9-b5zrt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigov251107me410shcf9af2facoqc-oidc-wi-cluster/openshift-apiserver-6f6d6b88f9-b5zrt has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-08-25T11:19:43Z\n Ended: 2026-08-25T11:27:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p\", pod=\"ingress-operator-5c7c8cb448-8cqf4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p/ingress-operator-5c7c8cb448-8cqf4 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-08-25T11:19:43Z\n Ended: 2026-08-25T11:27:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p\", pod=\"cluster-storage-operator-55f8b9d749-f6jpd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p/cluster-storage-operator-55f8b9d749-f6jpd has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-08-25T11:19:43Z\n Ended: 2026-08-25T11:27:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p\", pod=\"csi-snapshot-controller-operator-b779dc57c-dvvv2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigrgeahe87bis0ukc0jhq92p044to-b9f1c2q0b1l8t4p/csi-snapshot-controller-operator-b779dc57c-dvvv2 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-08-25T11:20:43Z\n Ended: 2026-08-25T11:31:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih0fp1fmn6t6fkvkmpfvi3s509kvi-l3u6d4n8d9v0b0o\", pod=\"router-849cb6d568-v29p5\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih0fp1fmn6t6fkvkmpfvi3s509kvi-l3u6d4n8d9v0b0o/router-849cb6d568-v29p5 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-08-25T11:20:43Z\n Ended: 2026-08-25T11:30:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihho1a123mh2it2n7vctui881hqae-w3e6w3f0k2r0k3f\", pod=\"ingress-operator-869d954d45-zjgp9\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihho1a123mh2it2n7vctui881hqae-w3e6w3f0k2r0k3f/ingress-operator-869d954d45-zjgp9 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-08-25T11:20:44Z\n Ended: 2026-08-25T11:26:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihho1a123mh2it2n7vctui881hqae-w3e6w3f0k2r0k3f\", pod=\"dns-operator-57f67c6dd6-5llhc\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihho1a123mh2it2n7vctui881hqae-w3e6w3f0k2r0k3f/dns-operator-57f67c6dd6-5llhc has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-08-25T11:20:44Z\n Ended: 2026-08-25T11:30:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihho1a123mh2it2n7vctui881hqae-w3e6w3f0k2r0k3f\", pod=\"cluster-network-operator-56d944cdf4-4jmzb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihho1a123mh2it2n7vctui881hqae-w3e6w3f0k2r0k3f/cluster-network-operator-56d944cdf4-4jmzb has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:27:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l\", pod=\"dns-operator-7bb8484758-wq6pf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l/dns-operator-7bb8484758-wq6pf has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:27:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l\", pod=\"ingress-operator-67bb49d9f8-txgcd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l/ingress-operator-67bb49d9f8-txgcd has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:27:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih2lpap41tb37m3d5e0kjrh3jjumd-v20260901\", pod=\"cluster-network-operator-7cbf6895bf-9bh5v\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih2lpap41tb37m3d5e0kjrh3jjumd-v20260901/cluster-network-operator-7cbf6895bf-9bh5v has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:27:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhvcgi49scqvaqim6cattb486hhm-a8p1s8n1b6h5h6p\", pod=\"openshift-apiserver-65b67c685b-wsrvn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhvcgi49scqvaqim6cattb486hhm-a8p1s8n1b6h5h6p/openshift-apiserver-65b67c685b-wsrvn has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhs26bbe32b2vccmrv2fet56h328-a1x4k9z1m7q2p5b\", pod=\"ingress-operator-74795fb8c8-ks7sl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhs26bbe32b2vccmrv2fet56h328-a1x4k9z1m7q2p5b/ingress-operator-74795fb8c8-ks7sl has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:27:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhso6vshstvo53lrpih930g1l92g-sre-hcp-cluster\", pod=\"ingress-operator-7b7448bb88-6zb7c\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhso6vshstvo53lrpih930g1l92g-sre-hcp-cluster/ingress-operator-7b7448bb88-6zb7c has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-08-25T11:21:43Z\n Ended: 2026-08-25T11:27:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l\", pod=\"dns-operator-f554cd46f-kznkx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l/dns-operator-f554cd46f-kznkx has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-08-25T11:21:44Z\n Ended: 2026-08-25T11:27:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l\", pod=\"cluster-network-operator-69749475d4-ksxdv\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l/cluster-network-operator-69749475d4-ksxdv has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-08-25T11:21:44Z\n Ended: 2026-08-25T11:27:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigoilh0gkbc6gnpu9m8c8j992l9qo-g4f2e7b8q7i0m5c\", pod=\"kube-apiserver-7bf5897fbd-llw26\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigoilh0gkbc6gnpu9m8c8j992l9qo-g4f2e7b8q7i0m5c/kube-apiserver-7bf5897fbd-llw26 has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-08-25T11:21:44Z\n Ended: 2026-08-25T11:27:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih2lpap41tb37m3d5e0kjrh3jjumd-v20260901\", pod=\"ingress-operator-5984c78bc-qprwn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih2lpap41tb37m3d5e0kjrh3jjumd-v20260901/ingress-operator-5984c78bc-qprwn has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-08-25T11:21:44Z\n Ended: 2026-08-25T11:29:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdih2lpap41tb37m3d5e0kjrh3jjumd-v20260901\", pod=\"dns-operator-5c9cd96b99-t8sp9\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdih2lpap41tb37m3d5e0kjrh3jjumd-v20260901/dns-operator-5c9cd96b99-t8sp9 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-08-25T11:21:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l\", pod=\"cluster-image-registry-operator-799f4846c9-hsr8m\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigh8k98csmst57kjmi4q90sgs2rrv-p1f9b4x3p6l5s0l/cluster-image-registry-operator-799f4846c9-hsr8m has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-08-25T11:22:45Z\n Ended: 2026-08-25T11:28:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihifj6r0cdpo4eenhniqoepqkarqg-d1l0y1w8p6m0t0v\", pod=\"ingress-operator-5dbdc46f56-fc76w\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihifj6r0cdpo4eenhniqoepqkarqg-d1l0y1w8p6m0t0v/ingress-operator-5dbdc46f56-fc76w has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-08-25T11:22:45Z\n Ended: 2026-08-25T11:29:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhutt6egf8nkbaeanni91sjqjthb-w0v1j4o3n3d9j5t\", pod=\"ingress-operator-66d96f9d4c-42j9m\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhutt6egf8nkbaeanni91sjqjthb-w0v1j4o3n3d9j5t/ingress-operator-66d96f9d4c-42j9m has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-08-25T11:22:45Z\n Ended: 2026-08-25T11:31:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhvcgi49scqvaqim6cattb486hhm-a8p1s8n1b6h5h6p\", pod=\"router-c456f5fbc-cpx28\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhvcgi49scqvaqim6cattb486hhm-a8p1s8n1b6h5h6p/router-c456f5fbc-cpx28 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"dns-operator-56478c8694-r5thf\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/dns-operator-56478c8694-r5thf has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"csi-snapshot-controller-operator-85d4568778-zpnvq\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/csi-snapshot-controller-operator-85d4568778-zpnvq has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighetrfta0j8u1pa03dtl5k9q9d0n-a8c8t3q7i8y6n4z\", pod=\"openshift-route-controller-manager-548849d46-7ksmw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighetrfta0j8u1pa03dtl5k9q9d0n-a8c8t3q7i8y6n4z/openshift-route-controller-manager-548849d46-7ksmw has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhutt6egf8nkbaeanni91sjqjthb-w0v1j4o3n3d9j5t\", pod=\"azure-disk-csi-driver-operator-69b864d46c-nsqcn\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhutt6egf8nkbaeanni91sjqjthb-w0v1j4o3n3d9j5t/azure-disk-csi-driver-operator-69b864d46c-nsqcn has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigjtvr1u96r8k47ihdtfqsispp6gk-ephemeral-disk\", pod=\"network-node-identity-7f66fcbc7-jl89z\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigjtvr1u96r8k47ihdtfqsispp6gk-ephemeral-disk/network-node-identity-7f66fcbc7-jl89z has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihi1m0v5mh13qjfkn5df7i1q7fivq-l5t4f9d3i5i4l3v\", pod=\"azure-cloud-controller-manager-77fdc894b8-b6lwr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihi1m0v5mh13qjfkn5df7i1q7fivq-l5t4f9d3i5i4l3v/azure-cloud-controller-manager-77fdc894b8-b6lwr has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"cluster-version-operator-65df5db46b-tlhsl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/cluster-version-operator-65df5db46b-tlhsl has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"cluster-storage-operator-567d9c98b5-6sdl4\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/cluster-storage-operator-567d9c98b5-6sdl4 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"dns-operator-7647df8fc9-vx29n\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/dns-operator-7647df8fc9-vx29n has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihht90lpuf6ep0e3v2mivrdmdj41b-cilium-cl\", pod=\"network-node-identity-5cd8ddcdc-5xhcd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihht90lpuf6ep0e3v2mivrdmdj41b-cilium-cl/network-node-identity-5cd8ddcdc-5xhcd has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdig96jq9lrl1uj7d4bbvn1vv0tvn7n-i8z5u7n3r5s9m7x\", pod=\"openshift-controller-manager-7fc7cf7599-xdwld\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdig96jq9lrl1uj7d4bbvn1vv0tvn7n-i8z5u7n3r5s9m7x/openshift-controller-manager-7fc7cf7599-xdwld has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihks11c1ah6qljd4mi7jgu0gu4n2v-ea-listfdq7jq\", pod=\"openshift-apiserver-6b65f797cd-t68bj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihks11c1ah6qljd4mi7jgu0gu4n2v-ea-listfdq7jq/openshift-apiserver-6b65f797cd-t68bj has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhsl4qrml5mn6tj1fociu37kug12-l9m0t9e6f1g2x0z\", pod=\"openshift-apiserver-759c44dc9f-n2pwg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhsl4qrml5mn6tj1fociu37kug12-l9m0t9e6f1g2x0z/openshift-apiserver-759c44dc9f-n2pwg has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighetrfta0j8u1pa03dtl5k9q9d0n-a8c8t3q7i8y6n4z\", pod=\"openshift-controller-manager-9854d9b75-hlthh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighetrfta0j8u1pa03dtl5k9q9d0n-a8c8t3q7i8y6n4z/openshift-controller-manager-9854d9b75-hlthh has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:32:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"cluster-policy-controller-649687f75f-7pxlr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/cluster-policy-controller-649687f75f-7pxlr has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"ingress-operator-6545876586-9w5gg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/ingress-operator-6545876586-9w5gg has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdiht06nip65lnjsqasans87pbijccq-r3e9n8e0v6j4q6w\", pod=\"cluster-policy-controller-6c4d44dc98-frm2c\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdiht06nip65lnjsqasans87pbijccq-r3e9n8e0v6j4q6w/cluster-policy-controller-6c4d44dc98-frm2c has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:32:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s\", pod=\"cluster-network-operator-85f89f8fc4-4sq6t\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihfmqfe5p85hkrqrs8mjlc5eja6jc-i2i1n6g2o3d6n5s/cluster-network-operator-85f89f8fc4-4sq6t has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigos14te0s7edb7ob5g0vc9ploiqq-c2b0y6a1g3p3d2h\", pod=\"azure-file-csi-driver-operator-bdc6cd8fb-vkxwm\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigos14te0s7edb7ob5g0vc9ploiqq-c2b0y6a1g3p3d2h/azure-file-csi-driver-operator-bdc6cd8fb-vkxwm has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:29:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihv4p7b4sh7keln37oq2g0um64blt-private-kas\", pod=\"kube-apiserver-6fb9877758-ftskx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihv4p7b4sh7keln37oq2g0um64blt-private-kas/kube-apiserver-6fb9877758-ftskx has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n State: Resolved\n Started: 2026-08-25T11:23:44Z\n Ended: 2026-08-25T11:32:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i\", pod=\"azure-disk-csi-driver-controller-5f8c6c7c87-m4x62\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihhuv0ee4hrgbaa3e8v6e4l0g5vp2-i6a9q6l3u0w2l6i/azure-disk-csi-driver-controller-5f8c6c7c87-m4x62 has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n State: Resolved\n Started: 2026-08-25T11:24:42Z\n Ended: 2026-08-25T11:32:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdighetrfta0j8u1pa03dtl5k9q9d0n-a8c8t3q7i8y6n4z\", pod=\"multus-admission-controller-55c5df6f5f-xkh8d\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdighetrfta0j8u1pa03dtl5k9q9d0n-a8c8t3q7i8y6n4z/multus-admission-controller-55c5df6f5f-xkh8d has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n State: Resolved\n Started: 2026-08-25T11:24:42Z\n Ended: 2026-08-25T11:30:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihi9v807i18d9cojts3mvodckr2de-f9a5n9z1j4t3j2e\", pod=\"ingress-operator-547bc47776-h9jrq\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihi9v807i18d9cojts3mvodckr2de-f9a5n9z1j4t3j2e/ingress-operator-547bc47776-h9jrq has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n State: Resolved\n Started: 2026-08-25T11:24:42Z\n Ended: 2026-08-25T11:30:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihi1m9ldavl96j49nv4088qcmig2f-t6m4j9a7p4l7y2n\", pod=\"cluster-network-operator-7c7bcfc67c-qm7mw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihi1m9ldavl96j49nv4088qcmig2f-t6m4j9a7p4l7y2n/cluster-network-operator-7c7bcfc67c-qm7mw has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n State: Resolved\n Started: 2026-08-25T11:24:43Z\n Ended: 2026-08-25T11:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihi9v807i18d9cojts3mvodckr2de-f9a5n9z1j4t3j2e\", pod=\"cluster-image-registry-operator-dc7fbc54c-ntqk2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihi9v807i18d9cojts3mvodckr2de-f9a5n9z1j4t3j2e/cluster-image-registry-operator-dc7fbc54c-ntqk2 has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n State: Resolved\n Started: 2026-08-25T11:24:43Z\n Ended: 2026-08-25T11:30:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihi1m9ldavl96j49nv4088qcmig2f-t6m4j9a7p4l7y2n\", pod=\"ingress-operator-6c84f87648-5hfq2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihi1m9ldavl96j49nv4088qcmig2f-t6m4j9a7p4l7y2n/ingress-operator-6c84f87648-5hfq2 has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n State: Resolved\n Started: 2026-08-25T11:24:45Z\n Ended: 2026-08-25T11:30:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdigohasqldgia8f7o0a0h5i1fpok8s-m7a3r7h3q4f4n3a\", pod=\"kube-storage-version-migrator-56b69bc9c8-5xctr\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdigohasqldgia8f7o0a0h5i1fpok8s-m7a3r7h3q4f4n3a/kube-storage-version-migrator-56b69bc9c8-5xctr has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n State: Resolved\n Started: 2026-08-25T11:25:42Z\n Ended: 2026-08-25T11:32:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5488384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdihknfg29t9kvecdq95udbscb5i26m-cilium-cluster\", pod=\"router-68768c77b5-4h4f8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdihknfg29t9kvecdq95udbscb5i26m-cilium-cluster/router-68768c77b5-4h4f8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"18c2c753d000a2018011c006460afd34bffb9613616246ee188db0df12480f7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"48ff412681e68aef8f22697d9b84a39dbb80151cba4ee2ed83b622757bccfcd8","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ef6720\u003e: \n GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9b5f24b-4624-4d6f-9212-bf4a1fd09040\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9b5f24b-4624-4d6f-9212-bf4a1fd09040\",\n \"name\": \"a9b5f24b-4624-4d6f-9212-bf4a1fd09040\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:22.386559609Z\",\n \"endTime\": \"2026-08-25T11:24:22.932654617Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ef6720\u003e: \n GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9b5f24b-4624-4d6f-9212-bf4a1fd09040\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9b5f24b-4624-4d6f-9212-bf4a1fd09040\",\n \"name\": \"a9b5f24b-4624-4d6f-9212-bf4a1fd09040\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:22.386559609Z\",\n \"endTime\": \"2026-08-25T11:24:22.932654617Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1bff01157d4330b9ecb54860b083ac4666db3a6f04bbed56b0b8cc36a6ea6fc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"660c5eae4bcca0af6b2fe0cfd998feca24db60cd960d79c76beb62534b620fc5","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00049fd20\u003e: \n failed to create HCP cluster np-downgrade-gwfjf8: failed waiting for cluster=\"np-downgrade-gwfjf8\" in resourcegroup=\"rg-np-downgrade-gwfjf8-7kkgmzwhgxk9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"name\": \"5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.69418327Z\",\n \"endTime\": \"2026-08-25T11:23:43.19279554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00049fd20\u003e: \n failed to create HCP cluster np-downgrade-gwfjf8: failed waiting for cluster=\"np-downgrade-gwfjf8\" in resourcegroup=\"rg-np-downgrade-gwfjf8-7kkgmzwhgxk9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"name\": \"5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.69418327Z\",\n \"endTime\": \"2026-08-25T11:23:43.19279554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12a1aaebf551a02f7b6b7033999753bb4e7d2b0f9d00ac7c03c67219d973f16","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21ac09d680687e161d8598c0961b9a7c5686b7f6ec5d1b3b37ed379662911eeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"3be09b3efaa1d6639c2674bcbca81f107fd2e424dd0133709cfc55b04fc0a27c","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c8120\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-4fsszwhdw97q\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84fe97bd-1239-454a-8374-a673a404e584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84fe97bd-1239-454a-8374-a673a404e584\",\n \"name\": \"84fe97bd-1239-454a-8374-a673a404e584\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:38.770881546Z\",\n \"endTime\": \"2026-08-25T11:21:43.018048135Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c8120\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-4fsszwhdw97q\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84fe97bd-1239-454a-8374-a673a404e584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84fe97bd-1239-454a-8374-a673a404e584\",\n \"name\": \"84fe97bd-1239-454a-8374-a673a404e584\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:38.770881546Z\",\n \"endTime\": \"2026-08-25T11:21:43.018048135Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21e18bd1f898fc33ea07201fe8d9e4074286107ddf57d55167231428f602aa57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"1ccba13814040aca0b43b4df7ff1901195302261df9120296071d4873aa80760","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a1cce0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-994pfg4rbw2z\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"name\": \"f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:18.051164248Z\",\n \"endTime\": \"2026-08-25T11:23:23.080049738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, konnectivity-agent, catalog-operator, control-plane-pki-operator, openshift-apiserver, ignition-server-proxy, openshift-route-controller-manager, cluster-autoscaler, machine-approver, kube-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, openshift-controller-manager, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, cluster-image-registry-operator, olm-collect-profiles, certified-operators-catalog, cluster-node-tuning-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-scheduler, ignition-server, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a1cce0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-994pfg4rbw2z\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"name\": \"f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:18.051164248Z\",\n \"endTime\": \"2026-08-25T11:23:23.080049738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, konnectivity-agent, catalog-operator, control-plane-pki-operator, openshift-apiserver, ignition-server-proxy, openshift-route-controller-manager, cluster-autoscaler, machine-approver, kube-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, openshift-controller-manager, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, cluster-image-registry-operator, olm-collect-profiles, certified-operators-catalog, cluster-node-tuning-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-scheduler, ignition-server, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb52007d345fecf08b7c3e5921187b32bee560cfda2f40a385c2e2262d097a34","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"345a72be2cbe683b920a5fd79da9519bc3b2e9426115bb6a1cf7aa938deddcbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"alert","signature_id":"27c12cddda1a8af9ca8f43e94a34fd689832822c5edc234c8b00528b723b3ae1","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-25T11:52:21Z\n Severity: Sev3\n Labels: alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j5488384-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5488384/providers/microsoft.monitor/accounts/services-j5488384\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired"}},{"row_id":"35164b0e6adc6ed3316c37473def31a82f8c7a8c38004949fba0bde03168865f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"9e89748c5a442a7a3524b941a931bf754405a837847407c813dd70e1c49eea38","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-rvnjl8\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8a40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-rvnjl8: failed waiting for cluster=\"cp-ystream-upgrade-4-21-rvnjl8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-w4mcqjptjkqr\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df113e55-476b-4fd3-b5ff-1eee5784babd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df113e55-476b-4fd3-b5ff-1eee5784babd\",\n \"name\": \"df113e55-476b-4fd3-b5ff-1eee5784babd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:29.586071021Z\",\n \"endTime\": \"2026-08-25T11:23:33.086793566Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, cluster-network-operator, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-rvnjl8\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8a40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-rvnjl8: failed waiting for cluster=\"cp-ystream-upgrade-4-21-rvnjl8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-w4mcqjptjkqr\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df113e55-476b-4fd3-b5ff-1eee5784babd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df113e55-476b-4fd3-b5ff-1eee5784babd\",\n \"name\": \"df113e55-476b-4fd3-b5ff-1eee5784babd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:29.586071021Z\",\n \"endTime\": \"2026-08-25T11:23:33.086793566Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, cluster-network-operator, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5de8e49a6cf79c3378e05d65f496119a89783cbcaf52079740e89743dcbbe80d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"a9fc0ce0c34af3d53b841c0d03266056bd51cbfcda7aed993a0a0b308252dad0","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:460]: failed to create nodepool nps-420-422 at version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000990860\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-q7nw79\" in resourcegroup=\"rg-np-skip-minor-q7nw79-pbbzs5p6b8hh\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f55038eb-3f9a-4f5d-9582-ceea6e3d3359\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f55038eb-3f9a-4f5d-9582-ceea6e3d3359\",\n \"name\": \"f55038eb-3f9a-4f5d-9582-ceea6e3d3359\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:19:15.692970564Z\",\n \"endTime\": \"2026-08-25T11:38:22.964822621Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:460]: failed to create nodepool nps-420-422 at version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000990860\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-q7nw79\" in resourcegroup=\"rg-np-skip-minor-q7nw79-pbbzs5p6b8hh\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f55038eb-3f9a-4f5d-9582-ceea6e3d3359\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f55038eb-3f9a-4f5d-9582-ceea6e3d3359\",\n \"name\": \"f55038eb-3f9a-4f5d-9582-ceea6e3d3359\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:19:15.692970564Z\",\n \"endTime\": \"2026-08-25T11:38:22.964822621Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"606630fb887e5f99726ff39db4358aca186ee9e0cfabd8257824b261f3eef12e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"283564a0bebb53ed7bfe47fcf31d4874c7bd4b8c4a658e0aea01d9dd41b0242d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-h5hbtn\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018e6b80\u003e: \n failed to create HCP cluster cluster-zstream-4-22-h5hbtn: failed waiting for cluster=\"cluster-zstream-4-22-h5hbtn\" in resourcegroup=\"rg-zstream-upgrade-4-22-6qd4tdr9bh97\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/672f2511-e84b-4f2b-92be-767c90f623e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/672f2511-e84b-4f2b-92be-767c90f623e8\",\n \"name\": \"672f2511-e84b-4f2b-92be-767c90f623e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.334111057Z\",\n \"endTime\": \"2026-08-25T11:23:43.006484355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, cluster-network-operator, catalog-operator, hosted-cluster-config-operator, cluster-storage-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, cluster-version-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-h5hbtn\" with version 4.22.10 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018e6b80\u003e: \n failed to create HCP cluster cluster-zstream-4-22-h5hbtn: failed waiting for cluster=\"cluster-zstream-4-22-h5hbtn\" in resourcegroup=\"rg-zstream-upgrade-4-22-6qd4tdr9bh97\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/672f2511-e84b-4f2b-92be-767c90f623e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/672f2511-e84b-4f2b-92be-767c90f623e8\",\n \"name\": \"672f2511-e84b-4f2b-92be-767c90f623e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.334111057Z\",\n \"endTime\": \"2026-08-25T11:23:43.006484355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, cluster-network-operator, catalog-operator, hosted-cluster-config-operator, cluster-storage-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, cluster-version-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61fb02a8853afbda2dd5dea7c86a474e243c133cedea7356b1bf30834496e1ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"e3bec1f2cb1a89b8221c4f2e37ae1979157dfc028f4587eae931e2b14a591671","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-dcp4d25dqkhg/cluster-candidate-4-21-798vcj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000983060\u003e: \n failed to create HCP cluster cluster-candidate-4-21-798vcj: failed waiting for cluster=\"cluster-candidate-4-21-798vcj\" in resourcegroup=\"rg-candidate-4-21-dcp4d25dqkhg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8712e933-315a-47e7-9c7b-be7c3ff94f5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8712e933-315a-47e7-9c7b-be7c3ff94f5e\",\n \"name\": \"8712e933-315a-47e7-9c7b-be7c3ff94f5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:17.342611915Z\",\n \"endTime\": \"2026-08-25T11:21:23.049204132Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, olm-operator, dns-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-version-operator, cluster-storage-operator, cluster-image-registry-operator, certified-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-dcp4d25dqkhg/cluster-candidate-4-21-798vcj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000983060\u003e: \n failed to create HCP cluster cluster-candidate-4-21-798vcj: failed waiting for cluster=\"cluster-candidate-4-21-798vcj\" in resourcegroup=\"rg-candidate-4-21-dcp4d25dqkhg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8712e933-315a-47e7-9c7b-be7c3ff94f5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8712e933-315a-47e7-9c7b-be7c3ff94f5e\",\n \"name\": \"8712e933-315a-47e7-9c7b-be7c3ff94f5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:17.342611915Z\",\n \"endTime\": \"2026-08-25T11:21:23.049204132Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, olm-operator, dns-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-version-operator, cluster-storage-operator, cluster-image-registry-operator, certified-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6347f6220023541952df10a4bb2d8f6f31ef64ed95b3c4f7e8342bc14ff86155","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"643ac15ebd4f500d7cdfd1c31ab51409301a12ee67319d5fe193f4ec82a3edb7","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c14a40\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dzzxfwq4m7h4\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/622e655d-77f3-4820-ba45-6088df8bd8ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/622e655d-77f3-4820-ba45-6088df8bd8ef\",\n \"name\": \"622e655d-77f3-4820-ba45-6088df8bd8ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.728472182Z\",\n \"endTime\": \"2026-08-25T11:23:43.188839328Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c14a40\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dzzxfwq4m7h4\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/622e655d-77f3-4820-ba45-6088df8bd8ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/622e655d-77f3-4820-ba45-6088df8bd8ef\",\n \"name\": \"622e655d-77f3-4820-ba45-6088df8bd8ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.728472182Z\",\n \"endTime\": \"2026-08-25T11:23:43.188839328Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66da503310cd381f628fff362b3fe7a471701d13304b0a6062639752ee2e9da6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"a52aed8bdf10bcdf14463a386f5de9b79532d4654760f91890403f2b46a1a5f7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b325c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hcdb8hh7hcl9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\",\n \"name\": \"a36ee1da-d072-4335-bca2-f17838bce647\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.267992354Z\",\n \"endTime\": \"2026-08-25T11:23:53.017585492Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b325c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hcdb8hh7hcl9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\",\n \"name\": \"a36ee1da-d072-4335-bca2-f17838bce647\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.267992354Z\",\n \"endTime\": \"2026-08-25T11:23:53.017585492Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"74426a739e4f86dd5411cdf4cc53f0d913b028f0d225517b981efa72c5de466f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, dns-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"67b718b40b153f1cc8e0928ee0cfd86ed075b522781399133d7e7f5ed1ebfa86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"bbe4d66acf643736fa2f7c6c58f4e7114799ee91a201d8f3333899113347be58","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-b2v7l9ll2xj8/cluster-candidate-4-20-k2wzcm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004a2280\u003e: \n failed to create HCP cluster cluster-candidate-4-20-k2wzcm: failed waiting for cluster=\"cluster-candidate-4-20-k2wzcm\" in resourcegroup=\"rg-candidate-4-20-b2v7l9ll2xj8\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b418ef9-27b0-4dec-abe0-9d11ca69b2b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b418ef9-27b0-4dec-abe0-9d11ca69b2b3\",\n \"name\": \"9b418ef9-27b0-4dec-abe0-9d11ca69b2b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:28.285801063Z\",\n \"endTime\": \"2026-08-25T11:22:33.058166877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-b2v7l9ll2xj8/cluster-candidate-4-20-k2wzcm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004a2280\u003e: \n failed to create HCP cluster cluster-candidate-4-20-k2wzcm: failed waiting for cluster=\"cluster-candidate-4-20-k2wzcm\" in resourcegroup=\"rg-candidate-4-20-b2v7l9ll2xj8\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b418ef9-27b0-4dec-abe0-9d11ca69b2b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b418ef9-27b0-4dec-abe0-9d11ca69b2b3\",\n \"name\": \"9b418ef9-27b0-4dec-abe0-9d11ca69b2b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:28.285801063Z\",\n \"endTime\": \"2026-08-25T11:22:33.058166877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6b1f2cd3f9bf4647f91111b76800adbb55d6a7426d6a777e75d6ebb035294a72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"7a249edbd9da943618f89813673eaaae45051d2a658fe6c77b5dd31a98bcbc7a","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-84b6mc with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013e27e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-84b6mc: failed waiting for cluster=\"np-version-upgrade-cluster-84b6mc\" in resourcegroup=\"rg-np-version-upgrade-84b6mc-gmpx6gg76fwl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/982916cf-c7a4-4457-9644-cc6c3c47733f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/982916cf-c7a4-4457-9644-cc6c3c47733f\",\n \"name\": \"982916cf-c7a4-4457-9644-cc6c3c47733f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:39.292750025Z\",\n \"endTime\": \"2026-08-25T11:21:43.032949082Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, redhat-marketplace-catalog, cluster-version-operator, certified-operators-catalog, openshift-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, olm-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-84b6mc with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013e27e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-84b6mc: failed waiting for cluster=\"np-version-upgrade-cluster-84b6mc\" in resourcegroup=\"rg-np-version-upgrade-84b6mc-gmpx6gg76fwl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/982916cf-c7a4-4457-9644-cc6c3c47733f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/982916cf-c7a4-4457-9644-cc6c3c47733f\",\n \"name\": \"982916cf-c7a4-4457-9644-cc6c3c47733f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:39.292750025Z\",\n \"endTime\": \"2026-08-25T11:21:43.032949082Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, redhat-marketplace-catalog, cluster-version-operator, certified-operators-catalog, openshift-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, olm-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c5e23c54c15e1a02e2aede84f40b02cd431b95af1ad41ff951af4ee773753f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"6e775195179703ce3594578f28d014a111b897b75498606daa63da0196f2a8f5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-8hkrqg\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016be120\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8hkrqg: failed waiting for cluster=\"cluster-zstream-4-21-8hkrqg\" in resourcegroup=\"rg-zstream-upgrade-4-21-2v84b2nxgnmb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6021a1ac-f792-4aa4-8f8b-7731245b72da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6021a1ac-f792-4aa4-8f8b-7731245b72da\",\n \"name\": \"6021a1ac-f792-4aa4-8f8b-7731245b72da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.214184097Z\",\n \"endTime\": \"2026-08-25T11:23:43.134206895Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-8hkrqg\" with version 4.21.29 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016be120\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8hkrqg: failed waiting for cluster=\"cluster-zstream-4-21-8hkrqg\" in resourcegroup=\"rg-zstream-upgrade-4-21-2v84b2nxgnmb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6021a1ac-f792-4aa4-8f8b-7731245b72da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6021a1ac-f792-4aa4-8f8b-7731245b72da\",\n \"name\": \"6021a1ac-f792-4aa4-8f8b-7731245b72da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.214184097Z\",\n \"endTime\": \"2026-08-25T11:23:43.134206895Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"730a6912dd4f203f803e5bcfe3ca99369d4c94c350da9f2b22f96d456118e5d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"1c7a8ccc960575f74b5f63b9d0fee47fa26ccbebde3ae4de7a7bbd10588a7ab3","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edefa0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rxw4z4c4mjfj\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"name\": \"d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:37.648275001Z\",\n \"endTime\": \"2026-08-25T11:22:43.091522188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edefa0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rxw4z4c4mjfj\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"name\": \"d7f39717-bb95-4ba8-b531-5600d9435095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:37.648275001Z\",\n \"endTime\": \"2026-08-25T11:22:43.091522188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55058ba88f617860ddd7645f53374b164637eda1467294e44a5e2bd25e9411f0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7961e20bbe83f868b00cc03b811acdc8790401862462e810bd773e54e41c6b73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"313c6b86047e9e386b672d793f4b878d127f3c65a84326d4cc2a6231581f8bc6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eea240\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ktb8zbvjkhxv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8663375f-0c13-4f3e-8ef2-47bc6045ec35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8663375f-0c13-4f3e-8ef2-47bc6045ec35\",\n \"name\": \"8663375f-0c13-4f3e-8ef2-47bc6045ec35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:37.983795353Z\",\n \"endTime\": \"2026-08-25T11:21:43.01549807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eea240\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ktb8zbvjkhxv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8663375f-0c13-4f3e-8ef2-47bc6045ec35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8663375f-0c13-4f3e-8ef2-47bc6045ec35\",\n \"name\": \"8663375f-0c13-4f3e-8ef2-47bc6045ec35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:37.983795353Z\",\n \"endTime\": \"2026-08-25T11:21:43.01549807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7cf3c534e4edb0c7ff6a7401315b3c888d19fab3fa99f7143752504d21e7dffe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"2bdb39b170bc1309151a9f820a6a2455edec3be17d92a609195106d19a9d06dc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-k6tv8vdvt6xx/cluster-candidate-4-22-zlxjzv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018feb00\u003e: \n failed to create HCP cluster cluster-candidate-4-22-zlxjzv: failed waiting for cluster=\"cluster-candidate-4-22-zlxjzv\" in resourcegroup=\"rg-candidate-4-22-k6tv8vdvt6xx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"name\": \"7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.96838764Z\",\n \"endTime\": \"2026-08-25T11:23:43.158826266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-k6tv8vdvt6xx/cluster-candidate-4-22-zlxjzv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018feb00\u003e: \n failed to create HCP cluster cluster-candidate-4-22-zlxjzv: failed waiting for cluster=\"cluster-candidate-4-22-zlxjzv\" in resourcegroup=\"rg-candidate-4-22-k6tv8vdvt6xx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"name\": \"7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.96838764Z\",\n \"endTime\": \"2026-08-25T11:23:43.158826266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"11980404af14f04ae347cf052e220be11d0d2393fbd6aa0e5b7916303d89b95b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7d30ce71225cc837737f0423658ba8e6386c1685b60b5aa7f3cdd4584d296c4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"17d508ac7b58e43e34e867e2daa42e985d102b6efa1571d29f852cfe4e849659","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rjf7mvrnbkrl/cluster-hs-pre-v6djjz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e2440\u003e: \n failed to create HCP cluster cluster-hs-pre-v6djjz: failed waiting for cluster=\"cluster-hs-pre-v6djjz\" in resourcegroup=\"rg-hs-pre--rjf7mvrnbkrl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4553519a-3596-446e-b502-3f9919b48e5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4553519a-3596-446e-b502-3f9919b48e5a\",\n \"name\": \"4553519a-3596-446e-b502-3f9919b48e5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.208748312Z\",\n \"endTime\": \"2026-08-25T11:22:03.069102239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, csi-snapshot-controller-operator, cluster-autoscaler, cluster-version-operator, dns-operator, ignition-server, certified-operators-catalog, control-plane-pki-operator, cluster-network-operator, olm-operator, kube-scheduler, hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, redhat-marketplace-catalog, openshift-apiserver, konnectivity-agent, redhat-operators-catalog, cluster-node-tuning-operator, olm-collect-profiles, ignition-server-proxy, kube-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, community-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rjf7mvrnbkrl/cluster-hs-pre-v6djjz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e2440\u003e: \n failed to create HCP cluster cluster-hs-pre-v6djjz: failed waiting for cluster=\"cluster-hs-pre-v6djjz\" in resourcegroup=\"rg-hs-pre--rjf7mvrnbkrl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4553519a-3596-446e-b502-3f9919b48e5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4553519a-3596-446e-b502-3f9919b48e5a\",\n \"name\": \"4553519a-3596-446e-b502-3f9919b48e5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.208748312Z\",\n \"endTime\": \"2026-08-25T11:22:03.069102239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, csi-snapshot-controller-operator, cluster-autoscaler, cluster-version-operator, dns-operator, ignition-server, certified-operators-catalog, control-plane-pki-operator, cluster-network-operator, olm-operator, kube-scheduler, hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, redhat-marketplace-catalog, openshift-apiserver, konnectivity-agent, redhat-operators-catalog, cluster-node-tuning-operator, olm-collect-profiles, ignition-server-proxy, kube-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, community-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7f51a63aa1240778ded5d505af5530ac22ace0cc0e4274c2109c92f972b4bad1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"0502e7bea814f2dea497a2fefaef20aef7c9a61fc571ac4894bd795f56d24e38","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fac960\u003e: \n failed to create HCP cluster ea-listfdq7jq: failed waiting for cluster=\"ea-listfdq7jq\" in resourcegroup=\"ea-list-pcxkj62k7nc9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/830cd850-ca4f-4036-ae2b-16567f3cc3fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/830cd850-ca4f-4036-ae2b-16567f3cc3fa\",\n \"name\": \"830cd850-ca4f-4036-ae2b-16567f3cc3fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.933938807Z\",\n \"endTime\": \"2026-08-25T11:23:53.031016737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, catalog-operator, redhat-marketplace-catalog, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, olm-operator, redhat-operators-catalog, cluster-version-operator, packageserver, openshift-route-controller-manager, certified-operators-catalog, hosted-cluster-config-operator, cluster-image-registry-operator, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fac960\u003e: \n failed to create HCP cluster ea-listfdq7jq: failed waiting for cluster=\"ea-listfdq7jq\" in resourcegroup=\"ea-list-pcxkj62k7nc9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/830cd850-ca4f-4036-ae2b-16567f3cc3fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/830cd850-ca4f-4036-ae2b-16567f3cc3fa\",\n \"name\": \"830cd850-ca4f-4036-ae2b-16567f3cc3fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.933938807Z\",\n \"endTime\": \"2026-08-25T11:23:53.031016737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, catalog-operator, redhat-marketplace-catalog, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, olm-operator, redhat-operators-catalog, cluster-version-operator, packageserver, openshift-route-controller-manager, certified-operators-catalog, hosted-cluster-config-operator, cluster-image-registry-operator, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"881f6243f9cfc8e227ef6911df01f5b9b8ba05e3861ec1b0f56d0e745102f9a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"8f0d0cefb3fa490d8596c9a345660d00241341f6c661aad661f74425c82f2008","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"b2468c4e9d126aa56394e8ef9b7bce10bb3f3907f961e58663665d48e197b818","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-wwr7p9 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010126e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-wwr7p9: failed waiting for cluster=\"np-version-upgrade-cluster-wwr7p9\" in resourcegroup=\"rg-np-version-upgrade-wwr7p9-x586qgvl4vx2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae3d88bb-54a0-4b93-b6f9-87cac8bac62b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae3d88bb-54a0-4b93-b6f9-87cac8bac62b\",\n \"name\": \"ae3d88bb-54a0-4b93-b6f9-87cac8bac62b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:30.253062363Z\",\n \"endTime\": \"2026-08-25T11:22:33.132975169Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, cluster-storage-operator, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-wwr7p9 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010126e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-wwr7p9: failed waiting for cluster=\"np-version-upgrade-cluster-wwr7p9\" in resourcegroup=\"rg-np-version-upgrade-wwr7p9-x586qgvl4vx2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae3d88bb-54a0-4b93-b6f9-87cac8bac62b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae3d88bb-54a0-4b93-b6f9-87cac8bac62b\",\n \"name\": \"ae3d88bb-54a0-4b93-b6f9-87cac8bac62b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:30.253062363Z\",\n \"endTime\": \"2026-08-25T11:22:33.132975169Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, cluster-storage-operator, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"94a593073d9f7c995fd69c6442d43d2fb5e9d75f83174b1ec9829143a607e78d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"b2a39dd5fceed70e654ab8c04be8e55e67bd4e3e353048a2a56cb92b2721b5ad","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcd40\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-58qpk5nt74kc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f05abc28-b92a-4df8-b0e3-a18378ebd5cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f05abc28-b92a-4df8-b0e3-a18378ebd5cc\",\n \"name\": \"f05abc28-b92a-4df8-b0e3-a18378ebd5cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:40.088109009Z\",\n \"endTime\": \"2026-08-25T11:21:43.005912277Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcd40\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-58qpk5nt74kc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f05abc28-b92a-4df8-b0e3-a18378ebd5cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f05abc28-b92a-4df8-b0e3-a18378ebd5cc\",\n \"name\": \"f05abc28-b92a-4df8-b0e3-a18378ebd5cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:40.088109009Z\",\n \"endTime\": \"2026-08-25T11:21:43.005912277Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9deaa735a051408a3c56e6df46a97b5760f80c3cb5dfd4f5cd53bf754ef424f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"19ac66d7406c2cf544c19dab33599323ce924d5b29029bc616b279fbe8796bf1","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-g4wtj9 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8a620\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-g4wtj9: failed waiting for cluster=\"np-version-upgrade-cluster-g4wtj9\" in resourcegroup=\"rg-np-version-upgrade-g4wtj9-mrccxbqqnhhc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\",\n \"name\": \"7d40e3df-681f-4926-943f-226d7903f59e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.789915832Z\",\n \"endTime\": \"2026-08-25T11:23:43.181882794Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, azure-cloud-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-g4wtj9 with version 4.20.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8a620\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-g4wtj9: failed waiting for cluster=\"np-version-upgrade-cluster-g4wtj9\" in resourcegroup=\"rg-np-version-upgrade-g4wtj9-mrccxbqqnhhc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\",\n \"name\": \"7d40e3df-681f-4926-943f-226d7903f59e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.789915832Z\",\n \"endTime\": \"2026-08-25T11:23:43.181882794Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, azure-cloud-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3c03ff8c6f57d65dc5b13024ef220ab850aed54bb6437f3a0bfad2241929fe27","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, cluster-policy-controller, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2735a003b914a4efc1dfc9f4c715d5193e8ddfc8f47e0bf26e457d521ec4cb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"d33c4e76ff9a43fcb2716c265a586de405e0886f83620fe7fc6fe9c586dabbef","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-jmx49s\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019b2760\u003e: \n failed to create HCP cluster cluster-zstream-5-0-jmx49s: failed waiting for cluster=\"cluster-zstream-5-0-jmx49s\" in resourcegroup=\"rg-zstream-upgrade-5-0-j2n79nx59vfm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"name\": \"f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:27.608916079Z\",\n \"endTime\": \"2026-08-25T11:21:32.925137069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-jmx49s\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019b2760\u003e: \n failed to create HCP cluster cluster-zstream-5-0-jmx49s: failed waiting for cluster=\"cluster-zstream-5-0-jmx49s\" in resourcegroup=\"rg-zstream-upgrade-5-0-j2n79nx59vfm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"name\": \"f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:27.608916079Z\",\n \"endTime\": \"2026-08-25T11:21:32.925137069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e9bb62f46de99c0acdbb4abf9f88a23afd79f058d2b6721fedb3e87d7c10a712","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad47231a8e20d80a55a03019ed0c925015624c7afc0991f92f7ce6dde39542e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"9a4f67edde370308f7181e7ac96b3504189ee39d35011561ed75c7aba37ebe2d","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001696280\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j578jj9q95nl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\",\n \"name\": \"dd6036ee-637e-4111-981e-76356c491fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.42538851Z\",\n \"endTime\": \"2026-08-25T11:23:43.113756051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, ingress-operator, olm-operator, catalog-operator, packageserver, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001696280\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j578jj9q95nl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\",\n \"name\": \"dd6036ee-637e-4111-981e-76356c491fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.42538851Z\",\n \"endTime\": \"2026-08-25T11:23:43.113756051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, ingress-operator, olm-operator, catalog-operator, packageserver, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"910e6132d71017042ccb4ad8536b725ec097c3baf39c134fb4939471b7ea12a4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b297ad4293385b202c5db693b0e5626904e6225656ca2c7eca3627391e4b734c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"dcaf8967a5afa5f20ee5c32afe78d8acf920baee1936ceb3553c7140fba63e19","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a1e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-lfghflc9cww8\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e61d9222-14e5-475a-beb7-5c1acf372d40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e61d9222-14e5-475a-beb7-5c1acf372d40\",\n \"name\": \"e61d9222-14e5-475a-beb7-5c1acf372d40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.98554443Z\",\n \"endTime\": \"2026-08-25T11:23:43.194079743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, packageserver, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a1e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-lfghflc9cww8\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e61d9222-14e5-475a-beb7-5c1acf372d40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e61d9222-14e5-475a-beb7-5c1acf372d40\",\n \"name\": \"e61d9222-14e5-475a-beb7-5c1acf372d40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.98554443Z\",\n \"endTime\": \"2026-08-25T11:23:43.194079743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, packageserver, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb1da7f902649fb2b7f4f3b3b39e9fb875200d4811cff63bb46f0604951ec55e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"edd0549c792e30ee83baa44f1b97d25996781b695fc96efc50800b1d38ed59e2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffc140\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-ng8s2hdwlfh7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a15e78e-de40-4ef2-91b2-32493940aa2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a15e78e-de40-4ef2-91b2-32493940aa2b\",\n \"name\": \"4a15e78e-de40-4ef2-91b2-32493940aa2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.979123804Z\",\n \"endTime\": \"2026-08-25T11:23:43.145765165Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffc140\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-ng8s2hdwlfh7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a15e78e-de40-4ef2-91b2-32493940aa2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a15e78e-de40-4ef2-91b2-32493940aa2b\",\n \"name\": \"4a15e78e-de40-4ef2-91b2-32493940aa2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.979123804Z\",\n \"endTime\": \"2026-08-25T11:23:43.145765165Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bde4d61cfcf832c6cbc4f4daaa8c7a3669f6401039b6243c6beaa9cb76f1071f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"e27b3eb99bc3107ba8d48dd22285100fd839e258993d8e1be4af2f04735bbfd0","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0e4a0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-zxfgxgnlxdgl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8594d25b-0975-46dd-b9ea-b253b264de22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8594d25b-0975-46dd-b9ea-b253b264de22\",\n \"name\": \"8594d25b-0975-46dd-b9ea-b253b264de22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:30.568853468Z\",\n \"endTime\": \"2026-08-25T11:23:33.079000658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator, olm-operator, packageserver, hosted-cluster-config-operator, cluster-node-tuning-operator, catalog-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0e4a0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-zxfgxgnlxdgl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8594d25b-0975-46dd-b9ea-b253b264de22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8594d25b-0975-46dd-b9ea-b253b264de22\",\n \"name\": \"8594d25b-0975-46dd-b9ea-b253b264de22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:30.568853468Z\",\n \"endTime\": \"2026-08-25T11:23:33.079000658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator, olm-operator, packageserver, hosted-cluster-config-operator, cluster-node-tuning-operator, catalog-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c00e0a545e603ce5a3138e2637e6ac04cb2296a3da3622f0ea797abd41e1c0e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"f5dd8ef4e673ae5c7d050ce26d2eedefb62e36c6ff1f4f21e2c53c0420687784","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f6740\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kl8r766fxv28\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c7634d2-1f12-4630-a3ca-cc8119d8315e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c7634d2-1f12-4630-a3ca-cc8119d8315e\",\n \"name\": \"8c7634d2-1f12-4630-a3ca-cc8119d8315e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.888667072Z\",\n \"endTime\": \"2026-08-25T11:23:43.024864375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f6740\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kl8r766fxv28\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c7634d2-1f12-4630-a3ca-cc8119d8315e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c7634d2-1f12-4630-a3ca-cc8119d8315e\",\n \"name\": \"8c7634d2-1f12-4630-a3ca-cc8119d8315e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:40.888667072Z\",\n \"endTime\": \"2026-08-25T11:23:43.024864375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c394d461648762dbcfbc678584e20c84b4c9279037cfd7402894c028374ab826","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"0fc0765dde4e73e193cda291461bd615b21c604ca712b5db69f701c1f66f193e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00197a700\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m4xxhgr55ksw\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a43ecf60-0976-4b44-9d53-138e3c078e40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a43ecf60-0976-4b44-9d53-138e3c078e40\",\n \"name\": \"a43ecf60-0976-4b44-9d53-138e3c078e40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.779051388Z\",\n \"endTime\": \"2026-08-25T11:22:03.065928952Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00197a700\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m4xxhgr55ksw\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a43ecf60-0976-4b44-9d53-138e3c078e40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a43ecf60-0976-4b44-9d53-138e3c078e40\",\n \"name\": \"a43ecf60-0976-4b44-9d53-138e3c078e40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.779051388Z\",\n \"endTime\": \"2026-08-25T11:22:03.065928952Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c832c04524233ee5fdf603f2434d3a63a6e67820cf9586253f7198e10b55da65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"dc93c011273628c3be4689097fa54bae4867b676ee6f408953d5289c92e59a88","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001372260\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b748z7nmqrc2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"name\": \"4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:48.323976798Z\",\n \"endTime\": \"2026-08-25T11:21:53.061758244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001372260\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b748z7nmqrc2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"name\": \"4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:48.323976798Z\",\n \"endTime\": \"2026-08-25T11:21:53.061758244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"951de8af83bafe4594b1180a34b1f825907eff4cdbe9d6ea0e8637301c989320","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb1c3220fe3da9106dbf5ba245975985ad3627e9cb90857d6a2ef8ff2c1f5eff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"822b64d629b6a8eaf1077aec1bbfe1cde30b863a89d30905f961017f5f5f8da1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6qqtw5xzzhgg/cluster-candidate-5-0-skdztd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae4220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-skdztd: failed waiting for cluster=\"cluster-candidate-5-0-skdztd\" in resourcegroup=\"rg-candidate-5-0-6qqtw5xzzhgg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"name\": \"bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.368910284Z\",\n \"endTime\": \"2026-08-25T11:22:03.073725302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, csi-snapshot-controller-operator, kube-storage-version-migrator, olm-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, community-operators-catalog, ingress-operator, catalog-operator, openshift-route-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6qqtw5xzzhgg/cluster-candidate-5-0-skdztd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae4220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-skdztd: failed waiting for cluster=\"cluster-candidate-5-0-skdztd\" in resourcegroup=\"rg-candidate-5-0-6qqtw5xzzhgg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"name\": \"bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.368910284Z\",\n \"endTime\": \"2026-08-25T11:22:03.073725302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, csi-snapshot-controller-operator, kube-storage-version-migrator, olm-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, community-operators-catalog, ingress-operator, catalog-operator, openshift-route-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6e365d99bf1fe7b4ecb0645e26216599f71568b4b9def38c7d2aa9626ca4060b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, kube-storage-version-migrator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb393e5a23fc438a53ff8f3a936a169a8e1ac0d4d67a4dc0cad022aa7d11b82b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"c3a84dd7ddae67604fc665cb0bbd0f847f461d2effc89e6c2b9a218c130289a2","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00151a360\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6hw5fkxk55cz\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c34c7baa-58c7-441d-9b4c-3aa764d9ea35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c34c7baa-58c7-441d-9b4c-3aa764d9ea35\",\n \"name\": \"c34c7baa-58c7-441d-9b4c-3aa764d9ea35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:29.99287713Z\",\n \"endTime\": \"2026-08-25T11:22:33.093324416Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00151a360\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6hw5fkxk55cz\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c34c7baa-58c7-441d-9b4c-3aa764d9ea35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c34c7baa-58c7-441d-9b4c-3aa764d9ea35\",\n \"name\": \"c34c7baa-58c7-441d-9b4c-3aa764d9ea35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:29.99287713Z\",\n \"endTime\": \"2026-08-25T11:22:33.093324416Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc72831f04c18b1250eb0934e01fc76ca734d1f696f7ef0fe19b089f4005c02d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"f6f2233f1410b658d6612e4e90347e1d367b288e857c9f69f6fd3585012f8aab","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007fc3a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-7qr482mpm7kt\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"name\": \"b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.045350362Z\",\n \"endTime\": \"2026-08-25T11:23:42.92215819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007fc3a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-7qr482mpm7kt\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"name\": \"b7872695-995d-44a9-aff2-649d7b0c3f98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.045350362Z\",\n \"endTime\": \"2026-08-25T11:23:42.92215819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ab3cc8ce178fa5fcf5b2293012368a7b35c8796690239f588635f09f67db92fe","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d4dfea7cb0dd508409a268c75a62df646caef0fa2132bfd9c300c300c92bedcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"319f6c3fb2a06c2f17d4aa28d324ad963b4199781ff328a4f220cea67272ca73","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14940\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-56n9dv9dj6vs\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"name\": \"877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:31.771881865Z\",\n \"endTime\": \"2026-08-25T11:24:32.975906881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, certified-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14940\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-56n9dv9dj6vs\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"name\": \"877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:31.771881865Z\",\n \"endTime\": \"2026-08-25T11:24:32.975906881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, certified-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52b927fb75e1a0e28c1c3ab5099caef3882ee435ade1e450f06515148c6a9d46","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dda0edafe63fe71b9109fea5b64b8ed616f618a947fd67a0539db8bd0a2e95b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"bac68d05e70928d803a2e96e39a3594853024ebe374dbc961bb7717d91c26078","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001224260\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2gp6hm88ddcx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c3733-f010-4f7e-948d-817202b134de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c3733-f010-4f7e-948d-817202b134de\",\n \"name\": \"f13c3733-f010-4f7e-948d-817202b134de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:31.739304195Z\",\n \"endTime\": \"2026-08-25T11:24:32.959602044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, openshift-route-controller-manager, cluster-storage-operator, catalog-operator, packageserver, kube-scheduler, openshift-apiserver, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-autoscaler, openshift-controller-manager, olm-operator, certified-operators-catalog, ingress-operator, konnectivity-agent, machine-approver, cluster-version-operator, olm-collect-profiles, azure-cloud-controller-manager, cluster-network-operator, dns-operator, cluster-image-registry-operator, cluster-policy-controller, hosted-cluster-config-operator, kube-storage-version-migrator, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001224260\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2gp6hm88ddcx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c3733-f010-4f7e-948d-817202b134de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c3733-f010-4f7e-948d-817202b134de\",\n \"name\": \"f13c3733-f010-4f7e-948d-817202b134de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:05:31.739304195Z\",\n \"endTime\": \"2026-08-25T11:24:32.959602044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, openshift-route-controller-manager, cluster-storage-operator, catalog-operator, packageserver, kube-scheduler, openshift-apiserver, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-autoscaler, openshift-controller-manager, olm-operator, certified-operators-catalog, ingress-operator, konnectivity-agent, machine-approver, cluster-version-operator, olm-collect-profiles, azure-cloud-controller-manager, cluster-network-operator, dns-operator, cluster-image-registry-operator, cluster-policy-controller, hosted-cluster-config-operator, kube-storage-version-migrator, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eaff31749bdea3d62d7c0ea0703d014aad9e49042c25ebdc0c47ab54c81ae3e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"946e42b1ec8ff2dfdb875eabb2e23ed3cccd0fb9157427c09f12a2be51c45b2b","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001336240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2zg2mjdfgmzm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489540be-0a79-4401-90e2-c83238026963\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489540be-0a79-4401-90e2-c83238026963\",\n \"name\": \"489540be-0a79-4401-90e2-c83238026963\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.053775952Z\",\n \"endTime\": \"2026-08-25T11:23:43.202927029Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, community-operators-catalog, openshift-route-controller-manager, ingress-operator, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, hosted-cluster-config-operator, catalog-operator, redhat-marketplace-catalog, cluster-storage-operator, dns-operator, cluster-network-operator, cluster-image-registry-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001336240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2zg2mjdfgmzm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489540be-0a79-4401-90e2-c83238026963\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489540be-0a79-4401-90e2-c83238026963\",\n \"name\": \"489540be-0a79-4401-90e2-c83238026963\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:39.053775952Z\",\n \"endTime\": \"2026-08-25T11:23:43.202927029Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, community-operators-catalog, openshift-route-controller-manager, ingress-operator, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, hosted-cluster-config-operator, catalog-operator, redhat-marketplace-catalog, cluster-storage-operator, dns-operator, cluster-network-operator, cluster-image-registry-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f113daa92b08d23fe182e1d2ef34fd4e8a6c480521ac1f498ba3542b6c8565c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"4ca6751eeab4072d0b147aaed4435bad054118427342ce0f828034310536b0c2","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00134ebc0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7p6xhpjkj2d7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"name\": \"e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.498742711Z\",\n \"endTime\": \"2026-08-25T11:23:52.96330385Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-policy-controller, dns-operator, catalog-operator, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, packageserver, azure-cloud-controller-manager, openshift-controller-manager, certified-operators-catalog, olm-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00134ebc0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7p6xhpjkj2d7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"name\": \"e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:49.498742711Z\",\n \"endTime\": \"2026-08-25T11:23:52.96330385Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-policy-controller, dns-operator, catalog-operator, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, packageserver, azure-cloud-controller-manager, openshift-controller-manager, certified-operators-catalog, olm-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3eeae8cb14c18985e089952373ed74210f1e046b3b560436cbdbaa6dc0b87f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"cf304d6dfff36c6441a2fb53d72be066b0745f911a7fb1567e495cd2a83bfd4e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-ssd9kv\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a661a0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-ssd9kv: failed waiting for cluster=\"cluster-zstream-4-20-ssd9kv\" in resourcegroup=\"rg-zstream-upgrade-4-20-9qqpqzsl79cn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a401f1f7-37d5-4c48-9273-6d48eff0c7a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a401f1f7-37d5-4c48-9273-6d48eff0c7a0\",\n \"name\": \"a401f1f7-37d5-4c48-9273-6d48eff0c7a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.959728228Z\",\n \"endTime\": \"2026-08-25T11:23:43.108250186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-ssd9kv\" with version 4.20.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a661a0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-ssd9kv: failed waiting for cluster=\"cluster-zstream-4-20-ssd9kv\" in resourcegroup=\"rg-zstream-upgrade-4-20-9qqpqzsl79cn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a401f1f7-37d5-4c48-9273-6d48eff0c7a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a401f1f7-37d5-4c48-9273-6d48eff0c7a0\",\n \"name\": \"a401f1f7-37d5-4c48-9273-6d48eff0c7a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:38.959728228Z\",\n \"endTime\": \"2026-08-25T11:23:43.108250186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3fee298eda47456328b956b6a8a6252a0a7f5aeb4813e04f84613f1935ceb13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"8306a5db6f3a5a75eed1ac89929790d5c6dfaf67eb9701d7ccbd095c361aa3a6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26600\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-g94rgnmwffff\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"name\": \"00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.361104Z\",\n \"endTime\": \"2026-08-25T11:21:33.042291432Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e26600\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-g94rgnmwffff\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"name\": \"00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:28.361104Z\",\n \"endTime\": \"2026-08-25T11:21:33.042291432Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f1744105155154513ebbf4bd7117f8888fd7e8a5f221414547d7257b1ee3bbdd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc1dc6f8a869ffba72f97fef153d552b7553ed879ce0fe357ff95cca9c950aca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"bbf6b517c402b8dff8d7ee72ef0aee64132aec114ab4d1d13c39df47394f1e2f","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-j4v9fp\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaa980\u003e: \n failed to create HCP cluster np-noedge-j4v9fp: failed waiting for cluster=\"np-noedge-j4v9fp\" in resourcegroup=\"rg-np-noedge-j4v9fp-4r7jnknp4pwv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"name\": \"e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:30.442745031Z\",\n \"endTime\": \"2026-08-25T11:23:33.077388395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-j4v9fp\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaa980\u003e: \n failed to create HCP cluster np-noedge-j4v9fp: failed waiting for cluster=\"np-noedge-j4v9fp\" in resourcegroup=\"rg-np-noedge-j4v9fp-4r7jnknp4pwv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"name\": \"e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:04:30.442745031Z\",\n \"endTime\": \"2026-08-25T11:23:33.077388395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a169e99d8eb2639f0a1059675f96bc5d6a3645d7a85053031c7e676b4ad42f09","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fcc0372a73c274021705e762fbb7a67b2b58f99c5496e8192ef2f2b6cce78fdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"8a57d6dcc713db3bc6dcc797a3a204959181e3f6745d603dd5d1db0fbf1450de","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df13c0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hf9vt4xl7cvh\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2c0e2a4-0c60-4344-aad1-28e0efa81dc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2c0e2a4-0c60-4344-aad1-28e0efa81dc3\",\n \"name\": \"a2c0e2a4-0c60-4344-aad1-28e0efa81dc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.592272694Z\",\n \"endTime\": \"2026-08-25T11:22:03.020998968Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-storage-operator, ingress-operator, community-operators-catalog, csi-snapshot-controller-operator, catalog-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, redhat-operators-catalog, olm-operator, cluster-network-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df13c0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hf9vt4xl7cvh\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2c0e2a4-0c60-4344-aad1-28e0efa81dc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2c0e2a4-0c60-4344-aad1-28e0efa81dc3\",\n \"name\": \"a2c0e2a4-0c60-4344-aad1-28e0efa81dc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:58.592272694Z\",\n \"endTime\": \"2026-08-25T11:22:03.020998968Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-storage-operator, ingress-operator, community-operators-catalog, csi-snapshot-controller-operator, catalog-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, redhat-operators-catalog, olm-operator, cluster-network-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fddb5a3517254869420dca0796a8d5d0741b11f8f3f0c265fb30c2e19a44cc1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"c5a29c38a564aa794f98663d402b32a7d56cfc809fa2b5066f123163d7b250c5","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb0120\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-bntb7vm54fqr\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f4a7238-dfd3-42b2-8b5c-10dd1b445fb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f4a7238-dfd3-42b2-8b5c-10dd1b445fb3\",\n \"name\": \"9f4a7238-dfd3-42b2-8b5c-10dd1b445fb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:27.301468987Z\",\n \"endTime\": \"2026-08-25T11:21:32.988392269Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb0120\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-bntb7vm54fqr\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f4a7238-dfd3-42b2-8b5c-10dd1b445fb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f4a7238-dfd3-42b2-8b5c-10dd1b445fb3\",\n \"name\": \"9f4a7238-dfd3-42b2-8b5c-10dd1b445fb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:02:27.301468987Z\",\n \"endTime\": \"2026-08-25T11:21:32.988392269Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff25e08852a3ecb4fee48717802c289eab8acfa1e76ea060f7acc323f45def08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","failed_at":"e2e","signature_id":"27c6b10bf1757b4f9a4d1318c60bd484d2401ca40c9413ba0f14effcfc7efe38","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8c220\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-c2x56bx2tvvb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"name\": \"8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:09.304723426Z\",\n \"endTime\": \"2026-08-25T11:22:13.012989469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8c220\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-c2x56bx2tvvb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"name\": \"8707f304-4ff5-45ff-8354-0739ca14d80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T11:03:09.304723426Z\",\n \"endTime\": \"2026-08-25T11:22:13.012989469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c975965241af31c57ec44c3fff1102f75c167bb738954ccc985f5a6bb7a81cd2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":48,"failure_pattern_ids":["03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","11980404af14f04ae347cf052e220be11d0d2393fbd6aa0e5b7916303d89b95b","1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","3c03ff8c6f57d65dc5b13024ef220ab850aed54bb6437f3a0bfad2241929fe27","470aab7d7a3605e8633a637a70d37da16684c6630946576af6d433f851290640","52b927fb75e1a0e28c1c3ab5099caef3882ee435ade1e450f06515148c6a9d46","55058ba88f617860ddd7645f53374b164637eda1467294e44a5e2bd25e9411f0","6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","6e365d99bf1fe7b4ecb0645e26216599f71568b4b9def38c7d2aa9626ca4060b","74426a739e4f86dd5411cdf4cc53f0d913b028f0d225517b981efa72c5de466f","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","910e6132d71017042ccb4ad8536b725ec097c3baf39c134fb4939471b7ea12a4","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","951de8af83bafe4594b1180a34b1f825907eff4cdbe9d6ea0e8637301c989320","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","a12a1aaebf551a02f7b6b7033999753bb4e7d2b0f9d00ac7c03c67219d973f16","a169e99d8eb2639f0a1059675f96bc5d6a3645d7a85053031c7e676b4ad42f09","a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","ab3cc8ce178fa5fcf5b2293012368a7b35c8796690239f588635f09f67db92fe","b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","bb52007d345fecf08b7c3e5921187b32bee560cfda2f40a385c2e2262d097a34","bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","c975965241af31c57ec44c3fff1102f75c167bb738954ccc985f5a6bb7a81cd2","deacd16fd42c584f7532a98cc9b198caf769401900c4d495ba1223ef893fffc3","e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","e9bb62f46de99c0acdbb4abf9f88a23afd79f058d2b6721fedb3e87d7c10a712","f1744105155154513ebbf4bd7117f8888fd7e8a5f221414547d7257b1ee3bbdd","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":48,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191713148801025","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T10:02:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cfb374638b95501d3e909f94e53f7c9d6ba246a051a88f285db1d9d957c7230e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191713148801025","occurred_at":"2026-08-25T10:02:43Z","failed_at":"e2e","signature_id":"04481477e7547c1ab7c165b4ddca30acd8f9d43ad49b8ba03ce0070a336cbe84","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-lpd7ls\" to 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5c5a0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-lpd7ls\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nkpr7nplpjkn\" to finish updating: GET https://rp.j8801025.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-lpd7ls\" to 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5c5a0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-lpd7ls\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nkpr7nplpjkn\" to finish updating: GET https://rp.j8801025.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192182877294592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"2cc8be49f793a785b95a1632909779adeb445998","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:56:13Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d7de514ac4a3cd904235240a88e851130434b181e958984c30283655d7672fc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192182877294592","occurred_at":"2026-08-25T09:56:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6669,"pr_state":"open","pr_sha":"5c0ce63e7ccb9720b19764274e1ec873a56d9880","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:52:51Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a9e6ee719bb400bbea041235e971af202aef97e5fd6a9ef97e48405949917fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","occurred_at":"2026-08-25T09:52:51Z","failed_at":"e2e","signature_id":"4dcc3e0028de7e96928b0a1a88758b6efab2ee551db9968cdbece8675c668057","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0026c6318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-krw8vkxmtzpb\" to finish deleting: GET https://rp.j5592064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"name\": \"74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:56:49.253326502Z\",\n \"endTime\": \"2026-08-25T11:20:52.069849414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdi5csg0pn2q641bbc4hk7ubn3ksvmp still exists (deletion dispatched at 2026-08-25T10:56:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0026c6318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-krw8vkxmtzpb\" to finish deleting: GET https://rp.j5592064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"name\": \"74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:56:49.253326502Z\",\n \"endTime\": \"2026-08-25T11:20:52.069849414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdi5csg0pn2q641bbc4hk7ubn3ksvmp still exists (deletion dispatched at 2026-08-25T10:56:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c911e8f8d3b9b1c297ff5952b87b160ed616daed5ab61dc88f5b3cb4a92a817f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","occurred_at":"2026-08-25T09:52:51Z","failed_at":"e2e","signature_id":"0fe7f6172a7cc0e6e9770da225a113c760ce09d48125d6f7381c144d3873bcb3","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.200.17:33911-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.200.17:33911-\u003e172.30.0.10:53: i/o timeout","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06","failure_pattern":"failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"\u003curl\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4."}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187087024427008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"2cc8be49f793a785b95a1632909779adeb445998","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:47:22Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6b463c845a6ac82ba9c7163be4cc003c5861c32fe473c12d5153f2033f79fc0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187087024427008","occurred_at":"2026-08-25T09:47:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186487230566400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6657,"pr_state":"open","pr_sha":"1b73f10d55f0c84fbb4b8c32966559fa111d80fa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:45:00Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ef1096f28871b6b0356901576238031152fab3920a042b66e7332de2db08b6c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186487230566400","occurred_at":"2026-08-25T09:45:00Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092185517952077824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"24553e52052e1e113727f8329d351ffa95503cd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:41:08Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f6350dae0e90ab1d0c907963d115ff2a5a0bbae5e8dbbd196c736f4f70e58cea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092185517952077824","occurred_at":"2026-08-25T09:41:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186982993104896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:39:05Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"96554087fe8474e04060a89f9ce9c68360980de2fd4577967cc206c9340f0ccb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186982993104896","occurred_at":"2026-08-25T09:39:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092185468836777984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:34:45Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"47d0f8e27d7c52c69312b0601a8016dceea2b4a4ec77ecb97c20bc753d855767","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092185468836777984","occurred_at":"2026-08-25T09:34:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186297740300288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:32:33Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bf2d9caaff6f9bdee6da0366e0a7e55153092aabb586cc0cc8d8e54cc2be7845","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186297740300288","occurred_at":"2026-08-25T09:32:33Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"7eff23c12e4a99825ffd97ce671ba00e24716cc4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:29:48Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"34cc693d45838c5528b54e7902b13624991e4dbdb88e761242458ab6a82c1444","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","occurred_at":"2026-08-25T09:29:48Z","failed_at":"e2e","signature_id":"772bc28982c38b2a6bed58697050c102f7ccd329ca804037b2e6053a2681cbd4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-6pb5wbpxgt8x/cluster-candidate-4-21-mvzhwr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d0280\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mvzhwr: failed waiting for cluster=\"cluster-candidate-4-21-mvzhwr\" in resourcegroup=\"rg-candidate-4-21-6pb5wbpxgt8x\" to finish creating: GET https://rp.j0831744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93a57e4a-983c-402e-a03b-d4f6bdacf351\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93a57e4a-983c-402e-a03b-d4f6bdacf351\",\n \"name\": \"93a57e4a-983c-402e-a03b-d4f6bdacf351\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:41:58.090010693Z\",\n \"endTime\": \"2026-08-25T11:01:01.195580215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-6pb5wbpxgt8x/cluster-candidate-4-21-mvzhwr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d0280\u003e: \n failed to create HCP cluster cluster-candidate-4-21-mvzhwr: failed waiting for cluster=\"cluster-candidate-4-21-mvzhwr\" in resourcegroup=\"rg-candidate-4-21-6pb5wbpxgt8x\" to finish creating: GET https://rp.j0831744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93a57e4a-983c-402e-a03b-d4f6bdacf351\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93a57e4a-983c-402e-a03b-d4f6bdacf351\",\n \"name\": \"93a57e4a-983c-402e-a03b-d4f6bdacf351\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:41:58.090010693Z\",\n \"endTime\": \"2026-08-25T11:01:01.195580215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"855737e176f9b4be6effa85cf7d221860e567444d6a004cb96adf6ac706c4abc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","occurred_at":"2026-08-25T09:29:48Z","failed_at":"alert","signature_id":"6db69bc2fe95a94bfd8232f25627f1581b6967e549eb3db637637bd4cef65f38","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T10:44:13Z\n Ended: 2026-08-25T11:18:16Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0831744-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0831744-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187338678472704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:27:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"022566c04d49d99760fbb47685595287b37337d0e4bce98dbb49a4ae28567e18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187338678472704","occurred_at":"2026-08-25T09:27:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6673/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188071016534016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6673,"pr_state":"closed","pr_sha":"476ea911c90a6b0b4a8fd4bdf33bf18f33b65250","final_merged_sha":"476ea911c90a6b0b4a8fd4bdf33bf18f33b65250","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T09:26:19Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"daf442c01e45d6b85bd9ac6de7e9a6b946269d68a526be334590abbb0bc0d2c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6673/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188071016534016","occurred_at":"2026-08-25T09:26:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186630705123328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"24553e52052e1e113727f8329d351ffa95503cd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:26:03Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5bc969b43841771d82b04615b5be8cf905388b3152c5049fb70e88560642a52d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186630705123328","occurred_at":"2026-08-25T09:26:03Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092181736954793984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"24553e52052e1e113727f8329d351ffa95503cd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:25:20Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3e2ef54a67062cd8a1128e5f131c9927c5cff136ec7ef444e105b8a90bf6a411","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092181736954793984","occurred_at":"2026-08-25T09:25:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187238849843200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"5fdef9ddfd1ced2b8d1ba4a721a7a99e261c25b7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:22:01Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f16f1ab3eb0d641e444b8821b80cf4954f7619a6be4acde38d77a63558a51b62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187238849843200","occurred_at":"2026-08-25T09:22:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187717013082112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:16:21Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a455b8d231b6c9358c1e73221500605afcefedea92b947c011cf282ff1400201","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187717013082112","occurred_at":"2026-08-25T09:16:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092185354298724352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:15:29Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f5cb619e879d5c8a997f915a3cfb89cdedcb2f2d03d635522a9038347df6b8f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092185354298724352","occurred_at":"2026-08-25T09:15:29Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186223836663808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"24553e52052e1e113727f8329d351ffa95503cd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:11:07Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7cfcb354a122fd245888c99d14bff460b12ecf46d259f14208edacae62cc25f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092186223836663808","occurred_at":"2026-08-25T09:11:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187355459883008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"open","pr_sha":"22dc550ce0b651f59badbe20c7f7c689df44703a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:11:01Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2ccb4b5f119474f9ac4e73961efeb10910d7a3a12c7ad348b1cdd7985cd11665","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092187355459883008","occurred_at":"2026-08-25T09:11:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092177317307617280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:08:37Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fb7dbe7d4843b437f9768f1244b141b68137c826341548fec8ada59bdafbd413","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092177317307617280","occurred_at":"2026-08-25T09:08:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092176181049692160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:04:06Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6b41ae0b21ed28b3dab698ef3ddc540e067ee2dc71e07593acc1ad2b598c303e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092176181049692160","occurred_at":"2026-08-25T09:04:06Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092176018763681792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"24553e52052e1e113727f8329d351ffa95503cd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T09:03:22Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"61b92bb62371b90224adea52437357cc7c3eb6e09a7213023dce3ebdb091197a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092176018763681792","occurred_at":"2026-08-25T09:03:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092174435585888256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"80d64452982e312903c5d754e62569361f45fc63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:57:10Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"571ec6a296236ce7e4cfd032944f15239e4e2c185a1cacd1281fc7b013f1aa51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092174435585888256","occurred_at":"2026-08-25T08:57:10Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092172606122758144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"80d64452982e312903c5d754e62569361f45fc63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:49:54Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d27077590c403238ee8b83921c20d20f0aeec5e6a3e06905048c9ff27fa5a69a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092172606122758144","occurred_at":"2026-08-25T08:49:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092170638788661248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"80d64452982e312903c5d754e62569361f45fc63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:42:05Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"35c97bab230cf80e89894b299737b52b1bdf34517a7027ab7111d3b92d7a9dd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092170638788661248","occurred_at":"2026-08-25T08:42:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6671/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092169758051930112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6671,"pr_state":"closed","pr_sha":"aecbabad655b49d25bcb1f74e91583271be4ca94","final_merged_sha":"aecbabad655b49d25bcb1f74e91583271be4ca94","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T08:38:31Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c3beed06068e740879ef52ed80cae5e03afcc96268bdd22ec96f2f6ca306a432","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6671/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092169758051930112","occurred_at":"2026-08-25T08:38:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092168981921140736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"80d64452982e312903c5d754e62569361f45fc63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:35:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4ed4f63329fa7384daf7d8084c067af61cbe2bb0fa6814ebda3b26dc26480685","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092168981921140736","occurred_at":"2026-08-25T08:35:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092167957672103936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6670,"pr_state":"open","pr_sha":"80d64452982e312903c5d754e62569361f45fc63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:31:26Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fd29022f0e3e330dd82b48a68c0109fd42baa77fa328928293ff817f901ed6db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092167957672103936","occurred_at":"2026-08-25T08:31:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092163955655970816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:15:32Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a6c676380fa914b0483e54c78776cb2976d3f55b9d7a720ba5a1e7dea9fdaec6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092163955655970816","occurred_at":"2026-08-25T08:15:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092163634334535680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:14:15Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"330ba909f85c0374d0065cba7d1ad6ee267d83a689ea7098cee0bdac246fc626","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092163634334535680","occurred_at":"2026-08-25T08:14:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092163015330762752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:11:47Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2971fcb011fafba4c75da7bb99aa17ee14865a74ea6322a14f91db540a78d2a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092163015330762752","occurred_at":"2026-08-25T08:11:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092161885301379072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:07:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6c037797562daa15f59a5eae869b0c6da619653231d6f4b5b1bc78bcc4e5d928","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092161885301379072","occurred_at":"2026-08-25T08:07:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092161871338541056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T08:07:15Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dbdbdf89eb2b76a4e1a0a5b9f0fe2cf3688ecc4a8ac030a3a06f7228d8dd957f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092161871338541056","occurred_at":"2026-08-25T08:07:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092158585722114048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T07:54:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"af4e2fdfd082987d28a1c272c3495ef1718317eff91539b77b6255c30b470186","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092158585722114048","occurred_at":"2026-08-25T07:54:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092155838054535168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T07:43:16Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bf8077d5e8f49e4e1667a3d0de5bcab608be2134d7cffa13dcfd92d4ac92d1d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092155838054535168","occurred_at":"2026-08-25T07:43:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092147015625478144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6669,"pr_state":"open","pr_sha":"5c0ce63e7ccb9720b19764274e1ec873a56d9880","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T07:08:13Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"365f363674c2226d7e9b5f810c09d32f30ecfa2eb148d5371f72e87eff2ad6a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092147015625478144","occurred_at":"2026-08-25T07:08:13Z","failed_at":"e2e","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6638,"pr_state":"open","pr_sha":"c403afe8d857fa8c76e2da31fc1c9d3952ca11b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T07:07:02Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"09538e40b7e453be445be77b38413d5cbf07ebddb7bc17e3516ffdca3aec6e6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","failed_at":"alert","signature_id":"8c886dd26eb0284df24c3f192d77269e9116e884fbce7b0f37949a6fdc5f288d","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:31:29Z\n Ended: 2026-08-25T08:38:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"384d6acfd298c6cd2239ba767194182512a81db0ae026c63b9c361d84a5b7ac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","failed_at":"alert","signature_id":"1d97f68d37328cfbf8faf6e5077141064b5bf086d525d3989deb1486a0606173","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:17:26Z\n Ended: 2026-08-25T08:33:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T08:17:26Z\n Ended: 2026-08-25T08:36:25Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"4c5a4c73144f3603ada2d1218c7a149ad00a8c15c2fc5eb8fca3eb8a0e4ae9bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","failed_at":"e2e","signature_id":"6d98542146857385e0ad2f780a80a61651f234191e5005f754beacbb7ea51a34","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kqf9svj6lsq5/cluster-hs-pre-xwpsbd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8a8a0\u003e: \n failed to create HCP cluster cluster-hs-pre-xwpsbd: failed waiting for cluster=\"cluster-hs-pre-xwpsbd\" in resourcegroup=\"rg-hs-pre--kqf9svj6lsq5\" to finish creating: GET https://rp.j6292864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\",\n \"name\": \"f0314e9a-f076-4160-9b33-87341813da0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:56:21.373383208Z\",\n \"endTime\": \"2026-08-25T08:15:30.901546963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kqf9svj6lsq5/cluster-hs-pre-xwpsbd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8a8a0\u003e: \n failed to create HCP cluster cluster-hs-pre-xwpsbd: failed waiting for cluster=\"cluster-hs-pre-xwpsbd\" in resourcegroup=\"rg-hs-pre--kqf9svj6lsq5\" to finish creating: GET https://rp.j6292864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\",\n \"name\": \"f0314e9a-f076-4160-9b33-87341813da0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:56:21.373383208Z\",\n \"endTime\": \"2026-08-25T08:15:30.901546963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb99f26cabb7d804fa13cc034f7516c547f67a329bcca89183127f8a7ebbb4d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","failed_at":"alert","signature_id":"592b27ee04df02dab6c1e953326523943126040c3e195f04af0ccc2cb082c127","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:19:22Z\n Ended: 2026-08-25T08:37:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f2abcc4-a7f6-4094-8ae9-551ab00643de\"\n Description: Pod ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-08-25T08:19:22Z\n Ended: 2026-08-25T08:36:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f2abcc4-a7f6-4094-8ae9-551ab00643de\"\n Description: Pod ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146713346183168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T07:07:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146704450064384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-25T07:06:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146691078623232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"d5a9f482aee0f501c8ae8bda8cd06ffa03214aaf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T07:06:55Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"de5b599436ac2041101443e6d86b6fe7794140cc51fe74e50d94a2821210e6e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146691078623232","occurred_at":"2026-08-25T07:06:55Z","failed_at":"alert","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:22:04Z\n Ended: 2026-08-25T08:51:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8623232-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6629,"pr_state":"open","pr_sha":"c6d46781a7abcc4c551cdf9edbb601d3aee0cc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T07:06:53Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"852f96a38758cfaf1f975471671943dc418ff7e1e70335e1e6bdfb24a6e26c08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","failed_at":"e2e","signature_id":"0d69566e1be51af62861a571d03269f04a5db50727a3f7e9802a7d8a5eb2ef75","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fa7a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5wf7vp46fpk2\" to finish creating: GET https://rp.j6249984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb3dd7de-2ea8-41b7-8963-6ca228011069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb3dd7de-2ea8-41b7-8963-6ca228011069\",\n \"name\": \"eb3dd7de-2ea8-41b7-8963-6ca228011069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T08:07:43.594897017Z\",\n \"endTime\": \"2026-08-25T08:26:51.952575117Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fa7a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5wf7vp46fpk2\" to finish creating: GET https://rp.j6249984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb3dd7de-2ea8-41b7-8963-6ca228011069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb3dd7de-2ea8-41b7-8963-6ca228011069\",\n \"name\": \"eb3dd7de-2ea8-41b7-8963-6ca228011069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T08:07:43.594897017Z\",\n \"endTime\": \"2026-08-25T08:26:51.952575117Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb85852bf6dca87c34657a5c571bd431b25bfa06258de9736ecc7dd8624fc554","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","failed_at":"e2e","signature_id":"e77ed3272a3aa5a99a10d9c5ad6121981ac3eb73b228a8dd38fba0c6882dbb91","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec0ac0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-8pcm786z9kl7\" to finish creating: GET https://rp.j6249984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07285d13-3ea1-4146-88cc-72658c636e9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07285d13-3ea1-4146-88cc-72658c636e9b\",\n \"name\": \"07285d13-3ea1-4146-88cc-72658c636e9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T08:06:31.910071805Z\",\n \"endTime\": \"2026-08-25T08:25:31.930786343Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec0ac0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-8pcm786z9kl7\" to finish creating: GET https://rp.j6249984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07285d13-3ea1-4146-88cc-72658c636e9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/07285d13-3ea1-4146-88cc-72658c636e9b\",\n \"name\": \"07285d13-3ea1-4146-88cc-72658c636e9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T08:06:31.910071805Z\",\n \"endTime\": \"2026-08-25T08:25:31.930786343Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e932d68e6936af440ae7c0c47d230f6410eab9ae1532aeacff5b1db791599586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","failed_at":"alert","signature_id":"85a4cfecb0adb1b2747bda34b1b1285829950a4778f69e05021f9596834b1154","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s) (1 unknown, 4 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T08:17:44Z\n Ended: 2026-08-25T08:24:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6249984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdfu24s4pvl5a530q5gqgms7ekvmjjt-v20260901\", pod=\"router-f9564f577-ntlvg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdfu24s4pvl5a530q5gqgms7ekvmjjt-v20260901/router-f9564f577-ntlvg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"7eff23c12e4a99825ffd97ce671ba00e24716cc4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T06:51:54Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3879fb84d685e9ae22680d174b2c82ef29845515a1ddae08e52c77dc925c66cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","failed_at":"e2e","signature_id":"180ef015a082a3e691d738ac631464aaaf83f5b0ebf48a41b3ee60048cb89ac9","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110e740\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ghjz2jzcblzs\" to finish creating: GET https://rp.j7480448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/556c0413-25c1-4420-86cd-b5562eb5b2ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/556c0413-25c1-4420-86cd-b5562eb5b2ba\",\n \"name\": \"556c0413-25c1-4420-86cd-b5562eb5b2ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:43:34.764859985Z\",\n \"endTime\": \"2026-08-25T08:02:41.010331478Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110e740\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ghjz2jzcblzs\" to finish creating: GET https://rp.j7480448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/556c0413-25c1-4420-86cd-b5562eb5b2ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/556c0413-25c1-4420-86cd-b5562eb5b2ba\",\n \"name\": \"556c0413-25c1-4420-86cd-b5562eb5b2ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:43:34.764859985Z\",\n \"endTime\": \"2026-08-25T08:02:41.010331478Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6298a11ba4caa8612c392621aaace5fefe893868fed373909d3f60b2249cb631","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","failed_at":"e2e","signature_id":"4afdddbae344462c995de2251551a735a47164d0bd4bc3714ba035dce8cb03a0","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa85c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4pdvp7d98cjv\" to finish creating: GET https://rp.j7480448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38bbf22-3f09-449b-92bd-a2c77dd73dbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38bbf22-3f09-449b-92bd-a2c77dd73dbe\",\n \"name\": \"f38bbf22-3f09-449b-92bd-a2c77dd73dbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:44:06.292489025Z\",\n \"endTime\": \"2026-08-25T08:03:10.8621061Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa85c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4pdvp7d98cjv\" to finish creating: GET https://rp.j7480448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38bbf22-3f09-449b-92bd-a2c77dd73dbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38bbf22-3f09-449b-92bd-a2c77dd73dbe\",\n \"name\": \"f38bbf22-3f09-449b-92bd-a2c77dd73dbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:44:06.292489025Z\",\n \"endTime\": \"2026-08-25T08:03:10.8621061Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c040aa3d57b375c32784460636f50a83a7b7c47ddc3c8a75131c65bba89b4568","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","failed_at":"e2e","signature_id":"02210cdb0477006b4ad2f63d27bbb22b1fec432dfc421b3f4927ea8331e71bd5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0006c4270\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[08:07:56.002]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:07:56.054]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[08:08:04.153]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:04.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[08:08:15.998]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:16.102]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0006c4270\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[08:07:56.002]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:07:56.054]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[08:08:04.153]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:04.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[08:08:15.998]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[08:08:16.102]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"bf87cfe5b41d04a13227b8cc9458e4b4b92a5088","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T06:27:13Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"6d1a34e49aa2fc8231680387dff2923add355d1f05b0c2af4a1ac1c48d729c65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"e2e","signature_id":"926bbbc4110caa69d86b22306ab7a9ad88bd051e3a6201884a30ef9e28c59b9b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-5h9dtgrjffg9/cluster-candidate-5-0-6hrjbx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001834220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-6hrjbx: failed waiting for cluster=\"cluster-candidate-5-0-6hrjbx\" in resourcegroup=\"rg-candidate-5-0-5h9dtgrjffg9\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47b8e57b-cd76-4c09-a1f3-3c1abc70b222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47b8e57b-cd76-4c09-a1f3-3c1abc70b222\",\n \"name\": \"47b8e57b-cd76-4c09-a1f3-3c1abc70b222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:14:43.287892255Z\",\n \"endTime\": \"2026-08-25T07:33:50.437023697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-5h9dtgrjffg9/cluster-candidate-5-0-6hrjbx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001834220\u003e: \n failed to create HCP cluster cluster-candidate-5-0-6hrjbx: failed waiting for cluster=\"cluster-candidate-5-0-6hrjbx\" in resourcegroup=\"rg-candidate-5-0-5h9dtgrjffg9\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47b8e57b-cd76-4c09-a1f3-3c1abc70b222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47b8e57b-cd76-4c09-a1f3-3c1abc70b222\",\n \"name\": \"47b8e57b-cd76-4c09-a1f3-3c1abc70b222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:14:43.287892255Z\",\n \"endTime\": \"2026-08-25T07:33:50.437023697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7a80571dad2c4d670d07ac1cfc3895bae947c33be0bda1a3e2e8fa653da06f55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"e2e","signature_id":"df56cc165c01e6c8c810f48db72eecd1abb1ce36621379d823c8e5cb3996a5ed","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00086b1c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-xsjvnxt7d74k\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6029c8d-c9f8-41ef-a520-4b8820105630\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6029c8d-c9f8-41ef-a520-4b8820105630\",\n \"name\": \"e6029c8d-c9f8-41ef-a520-4b8820105630\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:16:14.334902992Z\",\n \"endTime\": \"2026-08-25T07:35:20.361847152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00086b1c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-xsjvnxt7d74k\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6029c8d-c9f8-41ef-a520-4b8820105630\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6029c8d-c9f8-41ef-a520-4b8820105630\",\n \"name\": \"e6029c8d-c9f8-41ef-a520-4b8820105630\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:16:14.334902992Z\",\n \"endTime\": \"2026-08-25T07:35:20.361847152Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"83e82d1e2fe05c0d59cb290d20511af1ad5247b659087ef00a0119d96945c0c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"alert","signature_id":"7151c69efd10f0212e88001c4fa93856449529055b2b769b7fc63617db078de2","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T07:28:31Z\n Ended: 2026-08-25T07:46:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1244288-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdf5p07vju636qclvute8eqflcbovqk-n4c7o9t2g4r4v7k\", pod=\"router-bffff87c6-wgpsj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdf5p07vju636qclvute8eqflcbovqk-n4c7o9t2g4r4v7k/router-bffff87c6-wgpsj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"8c6f99a3c213246191c86b4a9dd2774cb035f9d363de3a9a0619829f00eb7255","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"e2e","signature_id":"ae0283cab4bf11ad3998038231a262c024b0853fa925e9504e4d72e900d671b8","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000812850\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[07:36:48.911]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:48.962]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[07:36:51.855]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:51.908]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[07:36:58.311]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:58.405]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000812850\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[07:36:48.911]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:48.962]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[07:36:51.855]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:51.908]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[07:36:58.311]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:36:58.405]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"8ed3e655c95da0f1006f251d504d6d564e25e579a7915794e3fa99986ac268ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"e2e","signature_id":"f9e5e4b9fd79fc6bdafcb10977e537bfeabb8f238b48fc177d8dc722a58ba115","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e409c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hnsmj56zlr79\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d4a286-1d08-44dc-8c95-6b2184c8c829\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d4a286-1d08-44dc-8c95-6b2184c8c829\",\n \"name\": \"c2d4a286-1d08-44dc-8c95-6b2184c8c829\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:16:14.438595425Z\",\n \"endTime\": \"2026-08-25T07:35:20.381767883Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e409c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hnsmj56zlr79\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d4a286-1d08-44dc-8c95-6b2184c8c829\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d4a286-1d08-44dc-8c95-6b2184c8c829\",\n \"name\": \"c2d4a286-1d08-44dc-8c95-6b2184c8c829\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:16:14.438595425Z\",\n \"endTime\": \"2026-08-25T07:35:20.381767883Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"92be2013eca61fd8183d1edd3e8975fdd454f8de231aecb3983fb6aead861a34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"e2e","signature_id":"ab35ad7b76b266586beb627cd857234f4deca4448950461a1483290474e1d37e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001ab3e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rcn4gmx4mkc4\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2a53eabb-5825-4532-a8be-6f9a8d4224ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2a53eabb-5825-4532-a8be-6f9a8d4224ca\",\n \"name\": \"2a53eabb-5825-4532-a8be-6f9a8d4224ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:14:44.590970562Z\",\n \"endTime\": \"2026-08-25T07:33:50.362043926Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001ab3e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rcn4gmx4mkc4\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2a53eabb-5825-4532-a8be-6f9a8d4224ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2a53eabb-5825-4532-a8be-6f9a8d4224ca\",\n \"name\": \"2a53eabb-5825-4532-a8be-6f9a8d4224ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:14:44.590970562Z\",\n \"endTime\": \"2026-08-25T07:33:50.362043926Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ca7fe348a10a7bd76e7aca57f64e90f925dbdf967957e9f545fcec31709596bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","failed_at":"e2e","signature_id":"2c553608c906b4a9046fc287e40a200fb56683ae155754190ceb3113e9c01826","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-9sc24ws2kvm7/cluster-candidate-4-22-tjqkdx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee8560\u003e: \n failed to create HCP cluster cluster-candidate-4-22-tjqkdx: failed waiting for cluster=\"cluster-candidate-4-22-tjqkdx\" in resourcegroup=\"rg-candidate-4-22-9sc24ws2kvm7\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7e0ad4e-0d4b-40fd-8921-744d4dcbc017\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7e0ad4e-0d4b-40fd-8921-744d4dcbc017\",\n \"name\": \"f7e0ad4e-0d4b-40fd-8921-744d4dcbc017\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:14:43.689975191Z\",\n \"endTime\": \"2026-08-25T07:33:50.401875542Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-9sc24ws2kvm7/cluster-candidate-4-22-tjqkdx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee8560\u003e: \n failed to create HCP cluster cluster-candidate-4-22-tjqkdx: failed waiting for cluster=\"cluster-candidate-4-22-tjqkdx\" in resourcegroup=\"rg-candidate-4-22-9sc24ws2kvm7\" to finish creating: GET https://rp.j1244288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7e0ad4e-0d4b-40fd-8921-744d4dcbc017\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7e0ad4e-0d4b-40fd-8921-744d4dcbc017\",\n \"name\": \"f7e0ad4e-0d4b-40fd-8921-744d4dcbc017\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:14:43.689975191Z\",\n \"endTime\": \"2026-08-25T07:33:50.401875542Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092131475158732800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T06:06:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a3edc1609d32704545c2a792ea7c980f9f03eaa1bb75aa01df83151781273a3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092131475158732800","occurred_at":"2026-08-25T06:06:28Z","failed_at":"e2e","signature_id":"8f69b74b7621ff0b0b8ce3121e6614ce9d3311a7090b53f7507107351ee1dd64","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000a22250\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[06:58:42.889]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:42.938]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[06:58:46.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:46.487]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[06:58:53.091]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:53.185]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000a22250\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[06:58:42.889]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:42.938]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[06:58:46.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:46.487]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[06:58:53.091]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:58:53.185]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm9448kb54lr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6433,"pr_state":"open","pr_sha":"9ec65054f149d1b7507171a2d613008ef1bc9761","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T06:04:04Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0706053017a1e09b8509babc84002a96cf92ceb0e0d93a85d43db99b9fffed9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","failed_at":"e2e","signature_id":"8d4d063103cc70a30037cefb230c7659ae5f161bfbe5aceb58476f2a4134c515","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bqwpq4nvz2hs\" to finish creating: GET https://rp.j9827200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f4ae122-44d7-4c73-aff5-38d3b3821a5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f4ae122-44d7-4c73-aff5-38d3b3821a5e\",\n \"name\": \"8f4ae122-44d7-4c73-aff5-38d3b3821a5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:09:27.254475963Z\",\n \"endTime\": \"2026-08-25T07:28:28.768093899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bqwpq4nvz2hs\" to finish creating: GET https://rp.j9827200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f4ae122-44d7-4c73-aff5-38d3b3821a5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f4ae122-44d7-4c73-aff5-38d3b3821a5e\",\n \"name\": \"8f4ae122-44d7-4c73-aff5-38d3b3821a5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:09:27.254475963Z\",\n \"endTime\": \"2026-08-25T07:28:28.768093899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0bdd5c36326d530b8c576bea8a99d18af3afad826e8deae9f0f63846c3cc3a8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","failed_at":"e2e","signature_id":"9ccbbc0b72f96cf97c329b914d4f5cc67ebf262aaacfa9b2db48024f1cbc3920","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d44a80\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-5kzvdf4f4k42\" to finish creating: GET https://rp.j9827200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d420474-93d9-441d-8bce-471c7c2b5fbc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d420474-93d9-441d-8bce-471c7c2b5fbc\",\n \"name\": \"9d420474-93d9-441d-8bce-471c7c2b5fbc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:09:25.661510376Z\",\n \"endTime\": \"2026-08-25T07:28:28.88206189Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d44a80\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-5kzvdf4f4k42\" to finish creating: GET https://rp.j9827200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d420474-93d9-441d-8bce-471c7c2b5fbc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9d420474-93d9-441d-8bce-471c7c2b5fbc\",\n \"name\": \"9d420474-93d9-441d-8bce-471c7c2b5fbc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:09:25.661510376Z\",\n \"endTime\": \"2026-08-25T07:28:28.88206189Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b30fe9f06329092075508982a0262f035920592ec46a1b14cd3f7ee588c99a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","failed_at":"e2e","signature_id":"09aaf44a85636c53922c8c55c3f077fa7ba9e0be3e43723e44ff0ee719184329","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0010b4150\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[07:29:52.744]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:44.6508715Z);\\nlet _endTime = datetime(2026-08-25T07:29:44.6508719Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:52.7254435Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f581084-3054-4981-a028-a6e365a00f11\\\",\\r\\n \\\"activityId\\\": \\\"86b66dc3-52a7-43ae-9446-0e6a169d09a6\\\",\\r\\n \\\"subActivityId\\\": \\\"a4b98a8a-4b86-4300-8041-f9a580f608a5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4759775b-7267-467e-8374-de7d5bbd8452\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f581084-3054-4981-a028-a6e365a00f11 ARID=86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.Http.CallContext/86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.ExecuteQuery/4759775b-7267-467e-8374-de7d5bbd8452 \\u003e DN.FE.ExecuteQuery/a4b98a8a-4b86-4300-8041-f9a580f608a5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:29:52.794]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:44.6508715Z);\\nlet _endTime = datetime(2026-08-25T07:29:44.6508719Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:52.7254435Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f581084-3054-4981-a028-a6e365a00f11\\\",\\r\\n \\\"activityId\\\": \\\"86b66dc3-52a7-43ae-9446-0e6a169d09a6\\\",\\r\\n \\\"subActivityId\\\": \\\"a4b98a8a-4b86-4300-8041-f9a580f608a5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4759775b-7267-467e-8374-de7d5bbd8452\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f581084-3054-4981-a028-a6e365a00f11 ARID=86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.Http.CallContext/86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.ExecuteQuery/4759775b-7267-467e-8374-de7d5bbd8452 \\u003e DN.FE.ExecuteQuery/a4b98a8a-4b86-4300-8041-f9a580f608a5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[07:29:56.028]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:52.8580255Z);\\nlet _endTime = datetime(2026-08-25T07:29:52.8580259Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:56.0095823Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa\\\",\\r\\n \\\"activityId\\\": \\\"c6d58314-2577-4cd2-b1d2-521b89903b88\\\",\\r\\n \\\"subActivityId\\\": \\\"9ee72512-2d51-4e94-951e-02405f520594\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2b025d9c-608c-4fc0-932a-9c4ff65e0033\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa ARID=c6d58314-2577-4cd2-b1d2-521b89903b88 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2b025d9c-608c-4fc0-932a-9c4ff65e0033 \\u003e DN.FE.ExecuteQuery/9ee72512-2d51-4e94-951e-02405f520594)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:29:56.254]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:52.8580255Z);\\nlet _endTime = datetime(2026-08-25T07:29:52.8580259Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:56.0095823Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa\\\",\\r\\n \\\"activityId\\\": \\\"c6d58314-2577-4cd2-b1d2-521b89903b88\\\",\\r\\n \\\"subActivityId\\\": \\\"9ee72512-2d51-4e94-951e-02405f520594\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2b025d9c-608c-4fc0-932a-9c4ff65e0033\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa ARID=c6d58314-2577-4cd2-b1d2-521b89903b88 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2b025d9c-608c-4fc0-932a-9c4ff65e0033 \\u003e DN.FE.ExecuteQuery/9ee72512-2d51-4e94-951e-02405f520594)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[07:30:33.649]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:30:25.5485738Z);\\nlet _endTime = datetime(2026-08-25T07:30:25.5485742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:30:33.6268189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;700462f9-237d-4475-9884-f9380159de6d\\\",\\r\\n \\\"activityId\\\": \\\"2ccd193c-b19d-4151-99b0-0e1bfdc6d835\\\",\\r\\n \\\"subActivityId\\\": \\\"456d8932-f4f5-43fd-b0b8-f104b6e33c9c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f51219a8-503f-4282-b78c-ed0028bde92a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;700462f9-237d-4475-9884-f9380159de6d ARID=2ccd193c-b19d-4151-99b0-0e1bfdc6d835 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f51219a8-503f-4282-b78c-ed0028bde92a \\u003e DN.FE.ExecuteQuery/456d8932-f4f5-43fd-b0b8-f104b6e33c9c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:30:33.755]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:30:25.5485738Z);\\nlet _endTime = datetime(2026-08-25T07:30:25.5485742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:30:33.6268189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;700462f9-237d-4475-9884-f9380159de6d\\\",\\r\\n \\\"activityId\\\": \\\"2ccd193c-b19d-4151-99b0-0e1bfdc6d835\\\",\\r\\n \\\"subActivityId\\\": \\\"456d8932-f4f5-43fd-b0b8-f104b6e33c9c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f51219a8-503f-4282-b78c-ed0028bde92a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;700462f9-237d-4475-9884-f9380159de6d ARID=2ccd193c-b19d-4151-99b0-0e1bfdc6d835 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f51219a8-503f-4282-b78c-ed0028bde92a \\u003e DN.FE.ExecuteQuery/456d8932-f4f5-43fd-b0b8-f104b6e33c9c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0010b4150\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[07:29:52.744]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:44.6508715Z);\\nlet _endTime = datetime(2026-08-25T07:29:44.6508719Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:52.7254435Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f581084-3054-4981-a028-a6e365a00f11\\\",\\r\\n \\\"activityId\\\": \\\"86b66dc3-52a7-43ae-9446-0e6a169d09a6\\\",\\r\\n \\\"subActivityId\\\": \\\"a4b98a8a-4b86-4300-8041-f9a580f608a5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4759775b-7267-467e-8374-de7d5bbd8452\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f581084-3054-4981-a028-a6e365a00f11 ARID=86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.Http.CallContext/86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.ExecuteQuery/4759775b-7267-467e-8374-de7d5bbd8452 \\u003e DN.FE.ExecuteQuery/a4b98a8a-4b86-4300-8041-f9a580f608a5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:29:52.794]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:44.6508715Z);\\nlet _endTime = datetime(2026-08-25T07:29:44.6508719Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:52.7254435Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f581084-3054-4981-a028-a6e365a00f11\\\",\\r\\n \\\"activityId\\\": \\\"86b66dc3-52a7-43ae-9446-0e6a169d09a6\\\",\\r\\n \\\"subActivityId\\\": \\\"a4b98a8a-4b86-4300-8041-f9a580f608a5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4759775b-7267-467e-8374-de7d5bbd8452\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f581084-3054-4981-a028-a6e365a00f11 ARID=86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.Http.CallContext/86b66dc3-52a7-43ae-9446-0e6a169d09a6 \\u003e GW.ExecuteQuery/4759775b-7267-467e-8374-de7d5bbd8452 \\u003e DN.FE.ExecuteQuery/a4b98a8a-4b86-4300-8041-f9a580f608a5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[07:29:56.028]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:52.8580255Z);\\nlet _endTime = datetime(2026-08-25T07:29:52.8580259Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:56.0095823Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa\\\",\\r\\n \\\"activityId\\\": \\\"c6d58314-2577-4cd2-b1d2-521b89903b88\\\",\\r\\n \\\"subActivityId\\\": \\\"9ee72512-2d51-4e94-951e-02405f520594\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2b025d9c-608c-4fc0-932a-9c4ff65e0033\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa ARID=c6d58314-2577-4cd2-b1d2-521b89903b88 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2b025d9c-608c-4fc0-932a-9c4ff65e0033 \\u003e DN.FE.ExecuteQuery/9ee72512-2d51-4e94-951e-02405f520594)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:29:56.254]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:29:52.8580255Z);\\nlet _endTime = datetime(2026-08-25T07:29:52.8580259Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:29:56.0095823Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa\\\",\\r\\n \\\"activityId\\\": \\\"c6d58314-2577-4cd2-b1d2-521b89903b88\\\",\\r\\n \\\"subActivityId\\\": \\\"9ee72512-2d51-4e94-951e-02405f520594\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2b025d9c-608c-4fc0-932a-9c4ff65e0033\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5a139d03-ea31-4fe0-b213-96a9c3436ffa ARID=c6d58314-2577-4cd2-b1d2-521b89903b88 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96a96db8-5e5c-470d-98e7-f595cc8873a0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2b025d9c-608c-4fc0-932a-9c4ff65e0033 \\u003e DN.FE.ExecuteQuery/9ee72512-2d51-4e94-951e-02405f520594)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[07:30:33.649]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:30:25.5485738Z);\\nlet _endTime = datetime(2026-08-25T07:30:25.5485742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:30:33.6268189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;700462f9-237d-4475-9884-f9380159de6d\\\",\\r\\n \\\"activityId\\\": \\\"2ccd193c-b19d-4151-99b0-0e1bfdc6d835\\\",\\r\\n \\\"subActivityId\\\": \\\"456d8932-f4f5-43fd-b0b8-f104b6e33c9c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f51219a8-503f-4282-b78c-ed0028bde92a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;700462f9-237d-4475-9884-f9380159de6d ARID=2ccd193c-b19d-4151-99b0-0e1bfdc6d835 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f51219a8-503f-4282-b78c-ed0028bde92a \\u003e DN.FE.ExecuteQuery/456d8932-f4f5-43fd-b0b8-f104b6e33c9c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[07:30:33.755]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:30:25.5485738Z);\\nlet _endTime = datetime(2026-08-25T07:30:25.5485742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-c86hfsjdphpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T07:30:33.6268189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;700462f9-237d-4475-9884-f9380159de6d\\\",\\r\\n \\\"activityId\\\": \\\"2ccd193c-b19d-4151-99b0-0e1bfdc6d835\\\",\\r\\n \\\"subActivityId\\\": \\\"456d8932-f4f5-43fd-b0b8-f104b6e33c9c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f51219a8-503f-4282-b78c-ed0028bde92a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;700462f9-237d-4475-9884-f9380159de6d ARID=2ccd193c-b19d-4151-99b0-0e1bfdc6d835 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/60f3c18c-8096-456b-8d2d-be70c3a82d69 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f51219a8-503f-4282-b78c-ed0028bde92a \\u003e DN.FE.ExecuteQuery/456d8932-f4f5-43fd-b0b8-f104b6e33c9c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"44b2e4efd95d3bd751da775cc35407b19ffc004d987cfda72e53123c679a3e6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","failed_at":"e2e","signature_id":"a71e16e9452ebbcb96b7843af9a8f55da51d63b8966fc237dc151347da86709b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f44480\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-pn894lhtsj9n\" to finish creating: GET https://rp.j9827200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f316fee-27a8-46da-955f-a2e939cbe4e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f316fee-27a8-46da-955f-a2e939cbe4e5\",\n \"name\": \"7f316fee-27a8-46da-955f-a2e939cbe4e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:10:29.36619422Z\",\n \"endTime\": \"2026-08-25T07:29:38.830703335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f44480\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-pn894lhtsj9n\" to finish creating: GET https://rp.j9827200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f316fee-27a8-46da-955f-a2e939cbe4e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f316fee-27a8-46da-955f-a2e939cbe4e5\",\n \"name\": \"7f316fee-27a8-46da-955f-a2e939cbe4e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:10:29.36619422Z\",\n \"endTime\": \"2026-08-25T07:29:38.830703335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee0d945f85cc846b1fff1ddb965c54186b20c3ea65dab509ee7f4f92253469f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","failed_at":"alert","signature_id":"9912e6cb8637cbc60f5d14947b0d4860ec095a6f064bb2b7e514dd1160ef1a11","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T07:21:49Z\n Ended: 2026-08-25T07:41:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9827200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdf3a6r28g6plkambnjikihao8o4b98-s6t4e3w5n3n3f4y\", pod=\"router-b5f575fc7-fx84w\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdf3a6r28g6plkambnjikihao8o4b98-s6t4e3w5n3n3f4y/router-b5f575fc7-fx84w has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-25T07:21:49Z\n Ended: 2026-08-25T07:38:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9827200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdf3pnsd1aqg1g8576apulo6qbhkmek-private-kas\", pod=\"router-bb7884476-jzjwj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdf3pnsd1aqg1g8576apulo6qbhkmek-private-kas/router-bb7884476-jzjwj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092122658786250752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6554,"pr_state":"open","pr_sha":"4e67bc77de1a80644494c4c53d6416cbb8167300","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T05:31:26Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0034b5b5fc17616c4595d33798aea47130f82cb3c58bf1eda9a132ab9ccaf62d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092122658786250752","occurred_at":"2026-08-25T05:31:26Z","failed_at":"e2e","signature_id":"0a451d1043de67f733809747c484d6352481472e46de0c7c6b4a4bb3a65e52d5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00118a160\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[06:47:25.114]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:22.1255109Z);\\nlet _endTime = datetime(2026-08-25T06:47:22.1255116Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:25.0967135Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c\\\",\\r\\n \\\"activityId\\\": \\\"7dd64d1c-c51d-4bda-a768-97e8b5ead7bf\\\",\\r\\n \\\"subActivityId\\\": \\\"06e1322c-145e-483e-8739-29dc51f280fb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9415089d-480a-4c1c-bb14-7fd15cb269d1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c ARID=7dd64d1c-c51d-4bda-a768-97e8b5ead7bf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9415089d-480a-4c1c-bb14-7fd15cb269d1 \\u003e DN.FE.ExecuteQuery/06e1322c-145e-483e-8739-29dc51f280fb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:47:25.165]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:22.1255109Z);\\nlet _endTime = datetime(2026-08-25T06:47:22.1255116Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:25.0967135Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c\\\",\\r\\n \\\"activityId\\\": \\\"7dd64d1c-c51d-4bda-a768-97e8b5ead7bf\\\",\\r\\n \\\"subActivityId\\\": \\\"06e1322c-145e-483e-8739-29dc51f280fb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9415089d-480a-4c1c-bb14-7fd15cb269d1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c ARID=7dd64d1c-c51d-4bda-a768-97e8b5ead7bf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9415089d-480a-4c1c-bb14-7fd15cb269d1 \\u003e DN.FE.ExecuteQuery/06e1322c-145e-483e-8739-29dc51f280fb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[06:47:28.557]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:25.2335043Z);\\nlet _endTime = datetime(2026-08-25T06:47:25.2335048Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:28.5392752Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;598e9d85-cc00-49e4-be80-597291b30526\\\",\\r\\n \\\"activityId\\\": \\\"30dd1538-d04d-4237-85d2-65955cf8a03b\\\",\\r\\n \\\"subActivityId\\\": \\\"70251134-ef12-4e91-8916-1feac4d77ba1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"93ec09c6-6033-4863-8dbf-9b9d8c6ce41b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;598e9d85-cc00-49e4-be80-597291b30526 ARID=30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.Http.CallContext/30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.ExecuteQuery/93ec09c6-6033-4863-8dbf-9b9d8c6ce41b \\u003e DN.FE.ExecuteQuery/70251134-ef12-4e91-8916-1feac4d77ba1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:47:28.606]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:25.2335043Z);\\nlet _endTime = datetime(2026-08-25T06:47:25.2335048Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:28.5392752Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;598e9d85-cc00-49e4-be80-597291b30526\\\",\\r\\n \\\"activityId\\\": \\\"30dd1538-d04d-4237-85d2-65955cf8a03b\\\",\\r\\n \\\"subActivityId\\\": \\\"70251134-ef12-4e91-8916-1feac4d77ba1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"93ec09c6-6033-4863-8dbf-9b9d8c6ce41b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;598e9d85-cc00-49e4-be80-597291b30526 ARID=30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.Http.CallContext/30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.ExecuteQuery/93ec09c6-6033-4863-8dbf-9b9d8c6ce41b \\u003e DN.FE.ExecuteQuery/70251134-ef12-4e91-8916-1feac4d77ba1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[06:47:34.946]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:31.8086106Z);\\nlet _endTime = datetime(2026-08-25T06:47:31.8086112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:34.9242569Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32\\\",\\r\\n \\\"activityId\\\": \\\"feb416da-c13a-4ccf-8869-2e872303ce63\\\",\\r\\n \\\"subActivityId\\\": \\\"8598484e-c376-4d81-8136-c32b1c69a7db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"541276a3-69a4-4660-b248-9e8d18f733a7\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32 ARID=feb416da-c13a-4ccf-8869-2e872303ce63 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/541276a3-69a4-4660-b248-9e8d18f733a7 \\u003e DN.FE.ExecuteQuery/8598484e-c376-4d81-8136-c32b1c69a7db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:47:35.049]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:31.8086106Z);\\nlet _endTime = datetime(2026-08-25T06:47:31.8086112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:34.9242569Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32\\\",\\r\\n \\\"activityId\\\": \\\"feb416da-c13a-4ccf-8869-2e872303ce63\\\",\\r\\n \\\"subActivityId\\\": \\\"8598484e-c376-4d81-8136-c32b1c69a7db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"541276a3-69a4-4660-b248-9e8d18f733a7\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32 ARID=feb416da-c13a-4ccf-8869-2e872303ce63 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/541276a3-69a4-4660-b248-9e8d18f733a7 \\u003e DN.FE.ExecuteQuery/8598484e-c376-4d81-8136-c32b1c69a7db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00118a160\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[06:47:25.114]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:22.1255109Z);\\nlet _endTime = datetime(2026-08-25T06:47:22.1255116Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:25.0967135Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c\\\",\\r\\n \\\"activityId\\\": \\\"7dd64d1c-c51d-4bda-a768-97e8b5ead7bf\\\",\\r\\n \\\"subActivityId\\\": \\\"06e1322c-145e-483e-8739-29dc51f280fb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9415089d-480a-4c1c-bb14-7fd15cb269d1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c ARID=7dd64d1c-c51d-4bda-a768-97e8b5ead7bf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9415089d-480a-4c1c-bb14-7fd15cb269d1 \\u003e DN.FE.ExecuteQuery/06e1322c-145e-483e-8739-29dc51f280fb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:47:25.165]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:22.1255109Z);\\nlet _endTime = datetime(2026-08-25T06:47:22.1255116Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:25.0967135Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c\\\",\\r\\n \\\"activityId\\\": \\\"7dd64d1c-c51d-4bda-a768-97e8b5ead7bf\\\",\\r\\n \\\"subActivityId\\\": \\\"06e1322c-145e-483e-8739-29dc51f280fb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9415089d-480a-4c1c-bb14-7fd15cb269d1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;80b0519e-2406-40eb-b816-cd534228b53c ARID=7dd64d1c-c51d-4bda-a768-97e8b5ead7bf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/96364a82-62ee-445d-9b0a-e40687bc35e5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9415089d-480a-4c1c-bb14-7fd15cb269d1 \\u003e DN.FE.ExecuteQuery/06e1322c-145e-483e-8739-29dc51f280fb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[06:47:28.557]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:25.2335043Z);\\nlet _endTime = datetime(2026-08-25T06:47:25.2335048Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:28.5392752Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;598e9d85-cc00-49e4-be80-597291b30526\\\",\\r\\n \\\"activityId\\\": \\\"30dd1538-d04d-4237-85d2-65955cf8a03b\\\",\\r\\n \\\"subActivityId\\\": \\\"70251134-ef12-4e91-8916-1feac4d77ba1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"93ec09c6-6033-4863-8dbf-9b9d8c6ce41b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;598e9d85-cc00-49e4-be80-597291b30526 ARID=30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.Http.CallContext/30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.ExecuteQuery/93ec09c6-6033-4863-8dbf-9b9d8c6ce41b \\u003e DN.FE.ExecuteQuery/70251134-ef12-4e91-8916-1feac4d77ba1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:47:28.606]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:25.2335043Z);\\nlet _endTime = datetime(2026-08-25T06:47:25.2335048Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:28.5392752Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;598e9d85-cc00-49e4-be80-597291b30526\\\",\\r\\n \\\"activityId\\\": \\\"30dd1538-d04d-4237-85d2-65955cf8a03b\\\",\\r\\n \\\"subActivityId\\\": \\\"70251134-ef12-4e91-8916-1feac4d77ba1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"93ec09c6-6033-4863-8dbf-9b9d8c6ce41b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;598e9d85-cc00-49e4-be80-597291b30526 ARID=30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.Http.CallContext/30dd1538-d04d-4237-85d2-65955cf8a03b \\u003e GW.ExecuteQuery/93ec09c6-6033-4863-8dbf-9b9d8c6ce41b \\u003e DN.FE.ExecuteQuery/70251134-ef12-4e91-8916-1feac4d77ba1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[06:47:34.946]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:31.8086106Z);\\nlet _endTime = datetime(2026-08-25T06:47:31.8086112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:34.9242569Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32\\\",\\r\\n \\\"activityId\\\": \\\"feb416da-c13a-4ccf-8869-2e872303ce63\\\",\\r\\n \\\"subActivityId\\\": \\\"8598484e-c376-4d81-8136-c32b1c69a7db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"541276a3-69a4-4660-b248-9e8d18f733a7\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32 ARID=feb416da-c13a-4ccf-8869-2e872303ce63 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/541276a3-69a4-4660-b248-9e8d18f733a7 \\u003e DN.FE.ExecuteQuery/8598484e-c376-4d81-8136-c32b1c69a7db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[06:47:35.049]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:47:31.8086106Z);\\nlet _endTime = datetime(2026-08-25T06:47:31.8086112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-q2kmlds2kptr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T06:47:34.9242569Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32\\\",\\r\\n \\\"activityId\\\": \\\"feb416da-c13a-4ccf-8869-2e872303ce63\\\",\\r\\n \\\"subActivityId\\\": \\\"8598484e-c376-4d81-8136-c32b1c69a7db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"541276a3-69a4-4660-b248-9e8d18f733a7\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bf820ce-7393-4166-a9e0-9c421ec12c32 ARID=feb416da-c13a-4ccf-8869-2e872303ce63 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/dea56255-2dda-40d2-87a2-e245e7c83acf \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/541276a3-69a4-4660-b248-9e8d18f733a7 \\u003e DN.FE.ExecuteQuery/8598484e-c376-4d81-8136-c32b1c69a7db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092109466840338432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6666,"pr_state":"closed","pr_sha":"a86004571d5c4fc79ff255fd53d013393120edbf","final_merged_sha":"a86004571d5c4fc79ff255fd53d013393120edbf","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T04:39:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092109160337379328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6667,"pr_state":"closed","pr_sha":"5d90e246870298020306251bf385d5e621ace8b3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T04:37:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"61cf00b7212fe1b718aff6c946372c783ff765af7b1b76477a397dd91f57d4ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092109160337379328","occurred_at":"2026-08-25T04:37:46Z","failed_at":"provision","signature_id":"bfc9e6d451ec941871c7fd17bc88e4377f14864e420626a1eb125b993b91f4d2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-08-25T05:18:44.021Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-08-25T05:19:25.373Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\\\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed to resolve 'login.microsoftonline.com' ([Errno -2] Name or service not known)\\\"))\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a135150bda94b29bbfd120583fd4db7aacc9a2d400307335c08f2083d06f63a","failure_pattern":"ERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed","search_query":"ERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\\\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed to resolve 'login.microsoftonline.com' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a135150bda94b29bbfd120583fd4db7aacc9a2d400307335c08f2083d06f63a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092101471658053632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T04:07:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ab460039f9cc9b4d84f19ec5cfe5cada0a2c4dc5e882cf825c3343f57c37c20b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092101471658053632","occurred_at":"2026-08-25T04:07:14Z","failed_at":"e2e","signature_id":"f34a1ababae19d28287c5f9ad6fadbb834adfab66885b9336868ab7e5fe3c4e0","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0017126b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[05:23:12.259]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:09.0740987Z);\\nlet _endTime = datetime(2026-08-25T05:23:09.0740993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:12.2406274Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927\\\",\\r\\n \\\"activityId\\\": \\\"67ccf73e-9d69-4840-9d95-7fc3c7426646\\\",\\r\\n \\\"subActivityId\\\": \\\"c397c476-48e7-486d-b8c6-5fe149482ad1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"156f2f9e-5c9d-4528-8d32-c99c4824c78f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927 ARID=67ccf73e-9d69-4840-9d95-7fc3c7426646 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/156f2f9e-5c9d-4528-8d32-c99c4824c78f \\u003e DN.FE.ExecuteQuery/c397c476-48e7-486d-b8c6-5fe149482ad1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[05:23:12.321]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:09.0740987Z);\\nlet _endTime = datetime(2026-08-25T05:23:09.0740993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:12.2406274Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927\\\",\\r\\n \\\"activityId\\\": \\\"67ccf73e-9d69-4840-9d95-7fc3c7426646\\\",\\r\\n \\\"subActivityId\\\": \\\"c397c476-48e7-486d-b8c6-5fe149482ad1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"156f2f9e-5c9d-4528-8d32-c99c4824c78f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927 ARID=67ccf73e-9d69-4840-9d95-7fc3c7426646 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/156f2f9e-5c9d-4528-8d32-c99c4824c78f \\u003e DN.FE.ExecuteQuery/c397c476-48e7-486d-b8c6-5fe149482ad1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[05:23:15.589]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:12.3880012Z);\\nlet _endTime = datetime(2026-08-25T05:23:12.3880018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:15.5708258Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c\\\",\\r\\n \\\"activityId\\\": \\\"4ef7f058-fd86-4bd2-9c91-414096eb303d\\\",\\r\\n \\\"subActivityId\\\": \\\"1b38b11c-2398-47e8-ab2c-841e6ed710a9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"1c09422b-21fa-40c4-89c0-61e4139c442e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c ARID=4ef7f058-fd86-4bd2-9c91-414096eb303d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/1c09422b-21fa-40c4-89c0-61e4139c442e \\u003e DN.FE.ExecuteQuery/1b38b11c-2398-47e8-ab2c-841e6ed710a9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[05:23:15.657]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:12.3880012Z);\\nlet _endTime = datetime(2026-08-25T05:23:12.3880018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:15.5708258Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c\\\",\\r\\n \\\"activityId\\\": \\\"4ef7f058-fd86-4bd2-9c91-414096eb303d\\\",\\r\\n \\\"subActivityId\\\": \\\"1b38b11c-2398-47e8-ab2c-841e6ed710a9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"1c09422b-21fa-40c4-89c0-61e4139c442e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c ARID=4ef7f058-fd86-4bd2-9c91-414096eb303d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/1c09422b-21fa-40c4-89c0-61e4139c442e \\u003e DN.FE.ExecuteQuery/1b38b11c-2398-47e8-ab2c-841e6ed710a9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[05:23:22.264]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:19.0401870Z);\\nlet _endTime = datetime(2026-08-25T05:23:19.0401876Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:22.2450735Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e\\\",\\r\\n \\\"activityId\\\": \\\"dfb66e31-a888-4332-80a1-2c81c1b9fd82\\\",\\r\\n \\\"subActivityId\\\": \\\"85c7b110-e789-4805-b1ad-e3b380f7d9dd\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9c069542-0b38-4e5c-8a35-6a188841ea44\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e ARID=dfb66e31-a888-4332-80a1-2c81c1b9fd82 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9c069542-0b38-4e5c-8a35-6a188841ea44 \\u003e DN.FE.ExecuteQuery/85c7b110-e789-4805-b1ad-e3b380f7d9dd)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[05:23:22.372]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:19.0401870Z);\\nlet _endTime = datetime(2026-08-25T05:23:19.0401876Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:22.2450735Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e\\\",\\r\\n \\\"activityId\\\": \\\"dfb66e31-a888-4332-80a1-2c81c1b9fd82\\\",\\r\\n \\\"subActivityId\\\": \\\"85c7b110-e789-4805-b1ad-e3b380f7d9dd\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9c069542-0b38-4e5c-8a35-6a188841ea44\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e ARID=dfb66e31-a888-4332-80a1-2c81c1b9fd82 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9c069542-0b38-4e5c-8a35-6a188841ea44 \\u003e DN.FE.ExecuteQuery/85c7b110-e789-4805-b1ad-e3b380f7d9dd)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0017126b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[05:23:12.259]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:09.0740987Z);\\nlet _endTime = datetime(2026-08-25T05:23:09.0740993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:12.2406274Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927\\\",\\r\\n \\\"activityId\\\": \\\"67ccf73e-9d69-4840-9d95-7fc3c7426646\\\",\\r\\n \\\"subActivityId\\\": \\\"c397c476-48e7-486d-b8c6-5fe149482ad1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"156f2f9e-5c9d-4528-8d32-c99c4824c78f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927 ARID=67ccf73e-9d69-4840-9d95-7fc3c7426646 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/156f2f9e-5c9d-4528-8d32-c99c4824c78f \\u003e DN.FE.ExecuteQuery/c397c476-48e7-486d-b8c6-5fe149482ad1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[05:23:12.321]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:09.0740987Z);\\nlet _endTime = datetime(2026-08-25T05:23:09.0740993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:12.2406274Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927\\\",\\r\\n \\\"activityId\\\": \\\"67ccf73e-9d69-4840-9d95-7fc3c7426646\\\",\\r\\n \\\"subActivityId\\\": \\\"c397c476-48e7-486d-b8c6-5fe149482ad1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"156f2f9e-5c9d-4528-8d32-c99c4824c78f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e4713658-d57e-46f4-8a0b-3d523d2d2927 ARID=67ccf73e-9d69-4840-9d95-7fc3c7426646 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/6b4fc1e1-15ba-4ca3-8fd3-10974ed553ab \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/156f2f9e-5c9d-4528-8d32-c99c4824c78f \\u003e DN.FE.ExecuteQuery/c397c476-48e7-486d-b8c6-5fe149482ad1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[05:23:15.589]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:12.3880012Z);\\nlet _endTime = datetime(2026-08-25T05:23:12.3880018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:15.5708258Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c\\\",\\r\\n \\\"activityId\\\": \\\"4ef7f058-fd86-4bd2-9c91-414096eb303d\\\",\\r\\n \\\"subActivityId\\\": \\\"1b38b11c-2398-47e8-ab2c-841e6ed710a9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"1c09422b-21fa-40c4-89c0-61e4139c442e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c ARID=4ef7f058-fd86-4bd2-9c91-414096eb303d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/1c09422b-21fa-40c4-89c0-61e4139c442e \\u003e DN.FE.ExecuteQuery/1b38b11c-2398-47e8-ab2c-841e6ed710a9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[05:23:15.657]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:12.3880012Z);\\nlet _endTime = datetime(2026-08-25T05:23:12.3880018Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:15.5708258Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c\\\",\\r\\n \\\"activityId\\\": \\\"4ef7f058-fd86-4bd2-9c91-414096eb303d\\\",\\r\\n \\\"subActivityId\\\": \\\"1b38b11c-2398-47e8-ab2c-841e6ed710a9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"1c09422b-21fa-40c4-89c0-61e4139c442e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e620926-f28e-4f81-bb34-776e2a4dc71c ARID=4ef7f058-fd86-4bd2-9c91-414096eb303d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/887aa0ee-f2c6-41e5-bc06-5009a86fede1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/1c09422b-21fa-40c4-89c0-61e4139c442e \\u003e DN.FE.ExecuteQuery/1b38b11c-2398-47e8-ab2c-841e6ed710a9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[05:23:22.264]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:19.0401870Z);\\nlet _endTime = datetime(2026-08-25T05:23:19.0401876Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:22.2450735Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e\\\",\\r\\n \\\"activityId\\\": \\\"dfb66e31-a888-4332-80a1-2c81c1b9fd82\\\",\\r\\n \\\"subActivityId\\\": \\\"85c7b110-e789-4805-b1ad-e3b380f7d9dd\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9c069542-0b38-4e5c-8a35-6a188841ea44\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e ARID=dfb66e31-a888-4332-80a1-2c81c1b9fd82 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9c069542-0b38-4e5c-8a35-6a188841ea44 \\u003e DN.FE.ExecuteQuery/85c7b110-e789-4805-b1ad-e3b380f7d9dd)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[05:23:22.372]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T05:23:19.0401870Z);\\nlet _endTime = datetime(2026-08-25T05:23:19.0401876Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8mfl5cspbkdx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T05:23:22.2450735Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e\\\",\\r\\n \\\"activityId\\\": \\\"dfb66e31-a888-4332-80a1-2c81c1b9fd82\\\",\\r\\n \\\"subActivityId\\\": \\\"85c7b110-e789-4805-b1ad-e3b380f7d9dd\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9c069542-0b38-4e5c-8a35-6a188841ea44\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c83e1791-3c16-4d2b-91a6-4658ad24736e ARID=dfb66e31-a888-4332-80a1-2c81c1b9fd82 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/606b6bc1-1f5e-4492-8e10-fe308d41c95c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9c069542-0b38-4e5c-8a35-6a188841ea44 \\u003e DN.FE.ExecuteQuery/85c7b110-e789-4805-b1ad-e3b380f7d9dd)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T03:45:56Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"05d95cb76ab5b2db572f43c89e63a8fad969d5d323823c6ab11d75ef3195e12f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"3163425eccfdd2023386c6da16d65b34dcdc351455d7c16151fd5399b7477ccd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","failed_at":"e2e","signature_id":"ec3713fc3fd2cd0078c515157ab16cbcdeafc43ac2e38c3ebeac3729eec40f03","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 618.602s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00014a160\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[04:54:21.505]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:13.5215143Z);\\nlet _endTime = datetime(2026-08-25T04:54:13.5215206Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:21.4861152Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a\\\",\\r\\n \\\"activityId\\\": \\\"d67ac573-0f51-4eb2-ac35-09b0ec3653e1\\\",\\r\\n \\\"subActivityId\\\": \\\"ab3166e7-5b0b-4155-b8e7-6abe1e54267e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"825fbb4b-a729-448a-ad52-44fe0cce83c2\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a ARID=d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.Http.CallContext/d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.ExecuteQuery/825fbb4b-a729-448a-ad52-44fe0cce83c2 \\u003e DN.FE.ExecuteQuery/ab3166e7-5b0b-4155-b8e7-6abe1e54267e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:54:21.554]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:13.5215143Z);\\nlet _endTime = datetime(2026-08-25T04:54:13.5215206Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:21.4861152Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a\\\",\\r\\n \\\"activityId\\\": \\\"d67ac573-0f51-4eb2-ac35-09b0ec3653e1\\\",\\r\\n \\\"subActivityId\\\": \\\"ab3166e7-5b0b-4155-b8e7-6abe1e54267e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"825fbb4b-a729-448a-ad52-44fe0cce83c2\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a ARID=d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.Http.CallContext/d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.ExecuteQuery/825fbb4b-a729-448a-ad52-44fe0cce83c2 \\u003e DN.FE.ExecuteQuery/ab3166e7-5b0b-4155-b8e7-6abe1e54267e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[04:54:24.629]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:21.6147104Z);\\nlet _endTime = datetime(2026-08-25T04:54:21.6147112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:24.6091269Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19\\\",\\r\\n \\\"activityId\\\": \\\"4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c\\\",\\r\\n \\\"subActivityId\\\": \\\"265d8ff4-1ace-44fe-88d7-7084889d640c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e0ed3c26-1203-4622-839a-062fcf54fd0b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19 ARID=4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e0ed3c26-1203-4622-839a-062fcf54fd0b \\u003e DN.FE.ExecuteQuery/265d8ff4-1ace-44fe-88d7-7084889d640c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:54:24.679]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:21.6147104Z);\\nlet _endTime = datetime(2026-08-25T04:54:21.6147112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:24.6091269Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19\\\",\\r\\n \\\"activityId\\\": \\\"4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c\\\",\\r\\n \\\"subActivityId\\\": \\\"265d8ff4-1ace-44fe-88d7-7084889d640c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e0ed3c26-1203-4622-839a-062fcf54fd0b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19 ARID=4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e0ed3c26-1203-4622-839a-062fcf54fd0b \\u003e DN.FE.ExecuteQuery/265d8ff4-1ace-44fe-88d7-7084889d640c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[04:54:31.005]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:28.0321387Z);\\nlet _endTime = datetime(2026-08-25T04:54:28.0321392Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:30.9871955Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c\\\",\\r\\n \\\"activityId\\\": \\\"b35e4e3a-fd10-4273-b73d-107f86af424a\\\",\\r\\n \\\"subActivityId\\\": \\\"2a561e63-03f6-46d1-8400-8aba70e601b4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"20ea4571-4aa1-4d90-8612-f4c8adf0d733\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c ARID=b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.Http.CallContext/b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.ExecuteQuery/20ea4571-4aa1-4d90-8612-f4c8adf0d733 \\u003e DN.FE.ExecuteQuery/2a561e63-03f6-46d1-8400-8aba70e601b4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:54:31.097]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:28.0321387Z);\\nlet _endTime = datetime(2026-08-25T04:54:28.0321392Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:30.9871955Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c\\\",\\r\\n \\\"activityId\\\": \\\"b35e4e3a-fd10-4273-b73d-107f86af424a\\\",\\r\\n \\\"subActivityId\\\": \\\"2a561e63-03f6-46d1-8400-8aba70e601b4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"20ea4571-4aa1-4d90-8612-f4c8adf0d733\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c ARID=b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.Http.CallContext/b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.ExecuteQuery/20ea4571-4aa1-4d90-8612-f4c8adf0d733 \\u003e DN.FE.ExecuteQuery/2a561e63-03f6-46d1-8400-8aba70e601b4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 618.602s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00014a160\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[04:54:21.505]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:13.5215143Z);\\nlet _endTime = datetime(2026-08-25T04:54:13.5215206Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:21.4861152Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a\\\",\\r\\n \\\"activityId\\\": \\\"d67ac573-0f51-4eb2-ac35-09b0ec3653e1\\\",\\r\\n \\\"subActivityId\\\": \\\"ab3166e7-5b0b-4155-b8e7-6abe1e54267e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"825fbb4b-a729-448a-ad52-44fe0cce83c2\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a ARID=d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.Http.CallContext/d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.ExecuteQuery/825fbb4b-a729-448a-ad52-44fe0cce83c2 \\u003e DN.FE.ExecuteQuery/ab3166e7-5b0b-4155-b8e7-6abe1e54267e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:54:21.554]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:13.5215143Z);\\nlet _endTime = datetime(2026-08-25T04:54:13.5215206Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:21.4861152Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a\\\",\\r\\n \\\"activityId\\\": \\\"d67ac573-0f51-4eb2-ac35-09b0ec3653e1\\\",\\r\\n \\\"subActivityId\\\": \\\"ab3166e7-5b0b-4155-b8e7-6abe1e54267e\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"825fbb4b-a729-448a-ad52-44fe0cce83c2\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0391bea9-3d54-4205-8ab5-51cb4fa2457a ARID=d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.Http.CallContext/d67ac573-0f51-4eb2-ac35-09b0ec3653e1 \\u003e GW.ExecuteQuery/825fbb4b-a729-448a-ad52-44fe0cce83c2 \\u003e DN.FE.ExecuteQuery/ab3166e7-5b0b-4155-b8e7-6abe1e54267e)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[04:54:24.629]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:21.6147104Z);\\nlet _endTime = datetime(2026-08-25T04:54:21.6147112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:24.6091269Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19\\\",\\r\\n \\\"activityId\\\": \\\"4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c\\\",\\r\\n \\\"subActivityId\\\": \\\"265d8ff4-1ace-44fe-88d7-7084889d640c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e0ed3c26-1203-4622-839a-062fcf54fd0b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19 ARID=4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e0ed3c26-1203-4622-839a-062fcf54fd0b \\u003e DN.FE.ExecuteQuery/265d8ff4-1ace-44fe-88d7-7084889d640c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:54:24.679]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:21.6147104Z);\\nlet _endTime = datetime(2026-08-25T04:54:21.6147112Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:24.6091269Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19\\\",\\r\\n \\\"activityId\\\": \\\"4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c\\\",\\r\\n \\\"subActivityId\\\": \\\"265d8ff4-1ace-44fe-88d7-7084889d640c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e0ed3c26-1203-4622-839a-062fcf54fd0b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edf1d2d3-10b4-4f32-a7f1-64c071125b19 ARID=4d35a4ea-993d-4ed5-bc53-a1db9f2fc22c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f9e96db5-f0f9-40f1-822b-1c1688a3f121 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e0ed3c26-1203-4622-839a-062fcf54fd0b \\u003e DN.FE.ExecuteQuery/265d8ff4-1ace-44fe-88d7-7084889d640c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[04:54:31.005]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:28.0321387Z);\\nlet _endTime = datetime(2026-08-25T04:54:28.0321392Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:30.9871955Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c\\\",\\r\\n \\\"activityId\\\": \\\"b35e4e3a-fd10-4273-b73d-107f86af424a\\\",\\r\\n \\\"subActivityId\\\": \\\"2a561e63-03f6-46d1-8400-8aba70e601b4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"20ea4571-4aa1-4d90-8612-f4c8adf0d733\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c ARID=b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.Http.CallContext/b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.ExecuteQuery/20ea4571-4aa1-4d90-8612-f4c8adf0d733 \\u003e DN.FE.ExecuteQuery/2a561e63-03f6-46d1-8400-8aba70e601b4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:54:31.097]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:54:28.0321387Z);\\nlet _endTime = datetime(2026-08-25T04:54:28.0321392Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xqbfw8gsxzsz'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:54:30.9871955Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c\\\",\\r\\n \\\"activityId\\\": \\\"b35e4e3a-fd10-4273-b73d-107f86af424a\\\",\\r\\n \\\"subActivityId\\\": \\\"2a561e63-03f6-46d1-8400-8aba70e601b4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"20ea4571-4aa1-4d90-8612-f4c8adf0d733\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;691bc226-47e5-48c8-a838-6ba3be72673c ARID=b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.Http.CallContext/b35e4e3a-fd10-4273-b73d-107f86af424a \\u003e GW.ExecuteQuery/20ea4571-4aa1-4d90-8612-f4c8adf0d733 \\u003e DN.FE.ExecuteQuery/2a561e63-03f6-46d1-8400-8aba70e601b4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"52dd3070d8cafd80aa3af8618bc7bb190554d37c5e7eedae4e972105f9814a95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","failed_at":"e2e","signature_id":"4654f560dff0807befbb619dbe35d2726f4f20abdaaa079b608640db38766779","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: failed to verify image registry is disabled on cluster disabled-image-registry-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc0012d80d8\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyBasicAccess failed: failed to list services: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: failed to verify image registry is disabled on cluster disabled-image-registry-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc0012d80d8\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyBasicAccess failed: failed to list services: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e43918bf51405cc433b6fba69bdc1ea9b482230892e03b1d2b08eabffb49ed22","failure_pattern":"VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout"}},{"row_id":"643d4d303f057d867e1ca63081d68ed6beb3ceeb729d6827cd2adc7ca35bde5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","failed_at":"e2e","signature_id":"ac5746c88f40ad3ccf73d0e708c6422fe20ba547abaaaef12f47d1492ae2b838","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaa540\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cfzqww2rkrtr\" to finish creating: GET https://rp.j7984128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f31d32d-9bbd-49ec-b81e-04190385bf80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f31d32d-9bbd-49ec-b81e-04190385bf80\",\n \"name\": \"2f31d32d-9bbd-49ec-b81e-04190385bf80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T04:28:47.291958618Z\",\n \"endTime\": \"2026-08-25T04:47:48.392085583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaa540\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cfzqww2rkrtr\" to finish creating: GET https://rp.j7984128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f31d32d-9bbd-49ec-b81e-04190385bf80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f31d32d-9bbd-49ec-b81e-04190385bf80\",\n \"name\": \"2f31d32d-9bbd-49ec-b81e-04190385bf80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T04:28:47.291958618Z\",\n \"endTime\": \"2026-08-25T04:47:48.392085583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ac5c8ab53938c52f8f56c8846987894360c182c72a6dc473789126a1a05f008","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","failed_at":"alert","signature_id":"939ee2aa813aeb34b5caee82729b4311d14f106923472e3fd6fed1c4ae71c667","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T04:41:18Z\n Ended: 2026-08-25T05:17:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7984128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdcnke2ne01uj8ljmgfd7q922scclq0-e3y5f4c0x3e4e4z\", pod=\"router-6b8bd96c4f-4slzz\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdcnke2ne01uj8ljmgfd7q922scclq0-e3y5f4c0x3e4e4z/router-6b8bd96c4f-4slzz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"96b6e30431c633ef19ad7f8f124daebbbfa5684c58c87a21041404d624556d6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","failed_at":"e2e","signature_id":"82fe3afdd0fba60d3c7679d4a315c8480ecd5cf974435df02861bd04baa9a825","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b20780\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-zmbgntqwnfcl\" to finish creating: GET https://rp.j7984128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51812137-4e69-43f4-b5f0-1ba75bc984cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51812137-4e69-43f4-b5f0-1ba75bc984cb\",\n \"name\": \"51812137-4e69-43f4-b5f0-1ba75bc984cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T04:28:42.421330013Z\",\n \"endTime\": \"2026-08-25T04:47:48.547435521Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b20780\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-zmbgntqwnfcl\" to finish creating: GET https://rp.j7984128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51812137-4e69-43f4-b5f0-1ba75bc984cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51812137-4e69-43f4-b5f0-1ba75bc984cb\",\n \"name\": \"51812137-4e69-43f4-b5f0-1ba75bc984cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T04:28:42.421330013Z\",\n \"endTime\": \"2026-08-25T04:47:48.547435521Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","e43918bf51405cc433b6fba69bdc1ea9b482230892e03b1d2b08eabffb49ed22","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092091165758197760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"54bb2db04ddf012dceee9c7b7444143151cf3825","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T03:26:17Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5f81b7018df88440166f4b1103fce0e422b0874fe45f1f4afd3596a090ecf28e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092091165758197760","occurred_at":"2026-08-25T03:26:17Z","failed_at":"e2e","signature_id":"0656eacbaaee9c8627f179735f527bbfa1a360407e229ef55d81deb432be6cbc","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0000d4c60\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[04:31:41.706]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:28.5213712Z);\\nlet _endTime = datetime(2026-08-25T04:31:28.5213718Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:41.6877827Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682\\\",\\r\\n \\\"activityId\\\": \\\"e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f\\\",\\r\\n \\\"subActivityId\\\": \\\"cb79ddc5-2e15-4ad7-b4a1-5e8175f94891\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682 ARID=e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.Http.CallContext/e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.ExecuteQuery/3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5 \\u003e DN.FE.ExecuteQuery/cb79ddc5-2e15-4ad7-b4a1-5e8175f94891)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:31:41.754]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:28.5213712Z);\\nlet _endTime = datetime(2026-08-25T04:31:28.5213718Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:41.6877827Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682\\\",\\r\\n \\\"activityId\\\": \\\"e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f\\\",\\r\\n \\\"subActivityId\\\": \\\"cb79ddc5-2e15-4ad7-b4a1-5e8175f94891\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682 ARID=e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.Http.CallContext/e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.ExecuteQuery/3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5 \\u003e DN.FE.ExecuteQuery/cb79ddc5-2e15-4ad7-b4a1-5e8175f94891)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[04:31:44.939]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:41.8336429Z);\\nlet _endTime = datetime(2026-08-25T04:31:41.8336434Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:44.9203710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350\\\",\\r\\n \\\"activityId\\\": \\\"666d0df5-25a1-4cfa-9916-e3823a7ba44e\\\",\\r\\n \\\"subActivityId\\\": \\\"8ded2343-8896-40df-9a04-f4d05531754b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"57be13cf-b001-4782-a859-f7bffcc24095\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350 ARID=666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.Http.CallContext/666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.ExecuteQuery/57be13cf-b001-4782-a859-f7bffcc24095 \\u003e DN.FE.ExecuteQuery/8ded2343-8896-40df-9a04-f4d05531754b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:31:44.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:41.8336429Z);\\nlet _endTime = datetime(2026-08-25T04:31:41.8336434Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:44.9203710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350\\\",\\r\\n \\\"activityId\\\": \\\"666d0df5-25a1-4cfa-9916-e3823a7ba44e\\\",\\r\\n \\\"subActivityId\\\": \\\"8ded2343-8896-40df-9a04-f4d05531754b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"57be13cf-b001-4782-a859-f7bffcc24095\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350 ARID=666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.Http.CallContext/666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.ExecuteQuery/57be13cf-b001-4782-a859-f7bffcc24095 \\u003e DN.FE.ExecuteQuery/8ded2343-8896-40df-9a04-f4d05531754b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[04:31:51.226]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:48.0313611Z);\\nlet _endTime = datetime(2026-08-25T04:31:48.0313617Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:51.2070563Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d\\\",\\r\\n \\\"activityId\\\": \\\"3c4a33bc-b463-4a6e-9b28-8d52ae986c82\\\",\\r\\n \\\"subActivityId\\\": \\\"57b721ec-0423-46f3-86e5-ae1e818f1557\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bc08fd46-3312-4b38-9391-708c1db998b5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d ARID=3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.Http.CallContext/3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.ExecuteQuery/bc08fd46-3312-4b38-9391-708c1db998b5 \\u003e DN.FE.ExecuteQuery/57b721ec-0423-46f3-86e5-ae1e818f1557)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:31:51.318]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:48.0313611Z);\\nlet _endTime = datetime(2026-08-25T04:31:48.0313617Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:51.2070563Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d\\\",\\r\\n \\\"activityId\\\": \\\"3c4a33bc-b463-4a6e-9b28-8d52ae986c82\\\",\\r\\n \\\"subActivityId\\\": \\\"57b721ec-0423-46f3-86e5-ae1e818f1557\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bc08fd46-3312-4b38-9391-708c1db998b5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d ARID=3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.Http.CallContext/3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.ExecuteQuery/bc08fd46-3312-4b38-9391-708c1db998b5 \\u003e DN.FE.ExecuteQuery/57b721ec-0423-46f3-86e5-ae1e818f1557)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0000d4c60\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[04:31:41.706]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:28.5213712Z);\\nlet _endTime = datetime(2026-08-25T04:31:28.5213718Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:41.6877827Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682\\\",\\r\\n \\\"activityId\\\": \\\"e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f\\\",\\r\\n \\\"subActivityId\\\": \\\"cb79ddc5-2e15-4ad7-b4a1-5e8175f94891\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682 ARID=e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.Http.CallContext/e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.ExecuteQuery/3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5 \\u003e DN.FE.ExecuteQuery/cb79ddc5-2e15-4ad7-b4a1-5e8175f94891)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:31:41.754]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:28.5213712Z);\\nlet _endTime = datetime(2026-08-25T04:31:28.5213718Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:41.6877827Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682\\\",\\r\\n \\\"activityId\\\": \\\"e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f\\\",\\r\\n \\\"subActivityId\\\": \\\"cb79ddc5-2e15-4ad7-b4a1-5e8175f94891\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;94be4808-db76-40dd-aaf6-ce4aad0d6682 ARID=e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.Http.CallContext/e9a1ad29-c17e-4936-8b9c-7ea9d7c6cc5f \\u003e GW.ExecuteQuery/3c54d0eb-f92a-4fe9-8d67-3ac81bfeeda5 \\u003e DN.FE.ExecuteQuery/cb79ddc5-2e15-4ad7-b4a1-5e8175f94891)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[04:31:44.939]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:41.8336429Z);\\nlet _endTime = datetime(2026-08-25T04:31:41.8336434Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:44.9203710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350\\\",\\r\\n \\\"activityId\\\": \\\"666d0df5-25a1-4cfa-9916-e3823a7ba44e\\\",\\r\\n \\\"subActivityId\\\": \\\"8ded2343-8896-40df-9a04-f4d05531754b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"57be13cf-b001-4782-a859-f7bffcc24095\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350 ARID=666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.Http.CallContext/666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.ExecuteQuery/57be13cf-b001-4782-a859-f7bffcc24095 \\u003e DN.FE.ExecuteQuery/8ded2343-8896-40df-9a04-f4d05531754b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:31:44.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:41.8336429Z);\\nlet _endTime = datetime(2026-08-25T04:31:41.8336434Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:44.9203710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350\\\",\\r\\n \\\"activityId\\\": \\\"666d0df5-25a1-4cfa-9916-e3823a7ba44e\\\",\\r\\n \\\"subActivityId\\\": \\\"8ded2343-8896-40df-9a04-f4d05531754b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"57be13cf-b001-4782-a859-f7bffcc24095\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;de7d3089-a049-4c03-b196-1ccd9497c350 ARID=666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.Http.CallContext/666d0df5-25a1-4cfa-9916-e3823a7ba44e \\u003e GW.ExecuteQuery/57be13cf-b001-4782-a859-f7bffcc24095 \\u003e DN.FE.ExecuteQuery/8ded2343-8896-40df-9a04-f4d05531754b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[04:31:51.226]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:48.0313611Z);\\nlet _endTime = datetime(2026-08-25T04:31:48.0313617Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:51.2070563Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d\\\",\\r\\n \\\"activityId\\\": \\\"3c4a33bc-b463-4a6e-9b28-8d52ae986c82\\\",\\r\\n \\\"subActivityId\\\": \\\"57b721ec-0423-46f3-86e5-ae1e818f1557\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bc08fd46-3312-4b38-9391-708c1db998b5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d ARID=3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.Http.CallContext/3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.ExecuteQuery/bc08fd46-3312-4b38-9391-708c1db998b5 \\u003e DN.FE.ExecuteQuery/57b721ec-0423-46f3-86e5-ae1e818f1557)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:31:51.318]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:31:48.0313611Z);\\nlet _endTime = datetime(2026-08-25T04:31:48.0313617Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-2hdph7ndrrzh'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:31:51.2070563Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d\\\",\\r\\n \\\"activityId\\\": \\\"3c4a33bc-b463-4a6e-9b28-8d52ae986c82\\\",\\r\\n \\\"subActivityId\\\": \\\"57b721ec-0423-46f3-86e5-ae1e818f1557\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bc08fd46-3312-4b38-9391-708c1db998b5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;900a470c-95cf-4b56-a536-ffebaed1de7d ARID=3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.Http.CallContext/3c4a33bc-b463-4a6e-9b28-8d52ae986c82 \\u003e GW.ExecuteQuery/bc08fd46-3312-4b38-9391-708c1db998b5 \\u003e DN.FE.ExecuteQuery/57b721ec-0423-46f3-86e5-ae1e818f1557)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092089899510403072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6660,"pr_state":"open","pr_sha":"a660b6e55fc0b5bbac8bc93588a1797519a9c76f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T03:21:15Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"22981d1dea5e84fc33b605de5bec99bf9a128954a80a2cade64762d6d27c57f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092089899510403072","occurred_at":"2026-08-25T03:21:15Z","failed_at":"e2e","signature_id":"b479792efcddc2bca8690264151fa877f3ce0d483876238bd57c9976805c7430","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c71900\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[04:25:11.894]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:08.5118208Z);\\nlet _endTime = datetime(2026-08-25T04:25:08.5118214Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:11.8759716Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e\\\",\\r\\n \\\"activityId\\\": \\\"71123125-c5bb-4db4-90e8-07275e6d8d7c\\\",\\r\\n \\\"subActivityId\\\": \\\"b347ccc1-e421-4a49-b038-2d4854509cad\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e8d7bd2f-af9b-4360-b688-da8755cdf7c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e ARID=71123125-c5bb-4db4-90e8-07275e6d8d7c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e8d7bd2f-af9b-4360-b688-da8755cdf7c3 \\u003e DN.FE.ExecuteQuery/b347ccc1-e421-4a49-b038-2d4854509cad)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:25:11.947]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:08.5118208Z);\\nlet _endTime = datetime(2026-08-25T04:25:08.5118214Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:11.8759716Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e\\\",\\r\\n \\\"activityId\\\": \\\"71123125-c5bb-4db4-90e8-07275e6d8d7c\\\",\\r\\n \\\"subActivityId\\\": \\\"b347ccc1-e421-4a49-b038-2d4854509cad\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e8d7bd2f-af9b-4360-b688-da8755cdf7c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e ARID=71123125-c5bb-4db4-90e8-07275e6d8d7c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e8d7bd2f-af9b-4360-b688-da8755cdf7c3 \\u003e DN.FE.ExecuteQuery/b347ccc1-e421-4a49-b038-2d4854509cad)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[04:25:15.243]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:12.0169012Z);\\nlet _endTime = datetime(2026-08-25T04:25:12.0169016Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:15.2248416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275\\\",\\r\\n \\\"activityId\\\": \\\"92fd7652-4336-4462-868c-12c3ba469e9d\\\",\\r\\n \\\"subActivityId\\\": \\\"81fcb6ea-8be7-40dd-8095-18a8ecb464b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"790c1312-33a0-482b-8940-5784463be54f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275 ARID=92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.Http.CallContext/92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.ExecuteQuery/790c1312-33a0-482b-8940-5784463be54f \\u003e DN.FE.ExecuteQuery/81fcb6ea-8be7-40dd-8095-18a8ecb464b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:25:15.308]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:12.0169012Z);\\nlet _endTime = datetime(2026-08-25T04:25:12.0169016Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:15.2248416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275\\\",\\r\\n \\\"activityId\\\": \\\"92fd7652-4336-4462-868c-12c3ba469e9d\\\",\\r\\n \\\"subActivityId\\\": \\\"81fcb6ea-8be7-40dd-8095-18a8ecb464b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"790c1312-33a0-482b-8940-5784463be54f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275 ARID=92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.Http.CallContext/92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.ExecuteQuery/790c1312-33a0-482b-8940-5784463be54f \\u003e DN.FE.ExecuteQuery/81fcb6ea-8be7-40dd-8095-18a8ecb464b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[04:25:24.336]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:20.0719077Z);\\nlet _endTime = datetime(2026-08-25T04:25:20.0719082Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:24.3156878Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74\\\",\\r\\n \\\"activityId\\\": \\\"354f4d7a-e0a3-4245-aead-8d5376f818a3\\\",\\r\\n \\\"subActivityId\\\": \\\"47228ea1-a92b-46c8-8e17-cb952ef7f8d3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"87aae106-3cd3-4084-a623-9299e4fdb101\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74 ARID=354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.Http.CallContext/354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.ExecuteQuery/87aae106-3cd3-4084-a623-9299e4fdb101 \\u003e DN.FE.ExecuteQuery/47228ea1-a92b-46c8-8e17-cb952ef7f8d3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:25:24.699]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:20.0719077Z);\\nlet _endTime = datetime(2026-08-25T04:25:20.0719082Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:24.3156878Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74\\\",\\r\\n \\\"activityId\\\": \\\"354f4d7a-e0a3-4245-aead-8d5376f818a3\\\",\\r\\n \\\"subActivityId\\\": \\\"47228ea1-a92b-46c8-8e17-cb952ef7f8d3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"87aae106-3cd3-4084-a623-9299e4fdb101\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74 ARID=354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.Http.CallContext/354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.ExecuteQuery/87aae106-3cd3-4084-a623-9299e4fdb101 \\u003e DN.FE.ExecuteQuery/47228ea1-a92b-46c8-8e17-cb952ef7f8d3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c71900\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[04:25:11.894]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:08.5118208Z);\\nlet _endTime = datetime(2026-08-25T04:25:08.5118214Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:11.8759716Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e\\\",\\r\\n \\\"activityId\\\": \\\"71123125-c5bb-4db4-90e8-07275e6d8d7c\\\",\\r\\n \\\"subActivityId\\\": \\\"b347ccc1-e421-4a49-b038-2d4854509cad\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e8d7bd2f-af9b-4360-b688-da8755cdf7c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e ARID=71123125-c5bb-4db4-90e8-07275e6d8d7c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e8d7bd2f-af9b-4360-b688-da8755cdf7c3 \\u003e DN.FE.ExecuteQuery/b347ccc1-e421-4a49-b038-2d4854509cad)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:25:11.947]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:08.5118208Z);\\nlet _endTime = datetime(2026-08-25T04:25:08.5118214Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:11.8759716Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e\\\",\\r\\n \\\"activityId\\\": \\\"71123125-c5bb-4db4-90e8-07275e6d8d7c\\\",\\r\\n \\\"subActivityId\\\": \\\"b347ccc1-e421-4a49-b038-2d4854509cad\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e8d7bd2f-af9b-4360-b688-da8755cdf7c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;89c18873-3158-47b2-8cc0-5b98350d4b0e ARID=71123125-c5bb-4db4-90e8-07275e6d8d7c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9047f797-93e4-4624-9e77-9a442f0115ff \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e8d7bd2f-af9b-4360-b688-da8755cdf7c3 \\u003e DN.FE.ExecuteQuery/b347ccc1-e421-4a49-b038-2d4854509cad)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[04:25:15.243]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:12.0169012Z);\\nlet _endTime = datetime(2026-08-25T04:25:12.0169016Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:15.2248416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275\\\",\\r\\n \\\"activityId\\\": \\\"92fd7652-4336-4462-868c-12c3ba469e9d\\\",\\r\\n \\\"subActivityId\\\": \\\"81fcb6ea-8be7-40dd-8095-18a8ecb464b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"790c1312-33a0-482b-8940-5784463be54f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275 ARID=92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.Http.CallContext/92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.ExecuteQuery/790c1312-33a0-482b-8940-5784463be54f \\u003e DN.FE.ExecuteQuery/81fcb6ea-8be7-40dd-8095-18a8ecb464b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:25:15.308]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:12.0169012Z);\\nlet _endTime = datetime(2026-08-25T04:25:12.0169016Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:15.2248416Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275\\\",\\r\\n \\\"activityId\\\": \\\"92fd7652-4336-4462-868c-12c3ba469e9d\\\",\\r\\n \\\"subActivityId\\\": \\\"81fcb6ea-8be7-40dd-8095-18a8ecb464b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"790c1312-33a0-482b-8940-5784463be54f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a23d2beb-5ad2-4517-8ea1-431e59258275 ARID=92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.Http.CallContext/92fd7652-4336-4462-868c-12c3ba469e9d \\u003e GW.ExecuteQuery/790c1312-33a0-482b-8940-5784463be54f \\u003e DN.FE.ExecuteQuery/81fcb6ea-8be7-40dd-8095-18a8ecb464b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[04:25:24.336]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:20.0719077Z);\\nlet _endTime = datetime(2026-08-25T04:25:20.0719082Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:24.3156878Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74\\\",\\r\\n \\\"activityId\\\": \\\"354f4d7a-e0a3-4245-aead-8d5376f818a3\\\",\\r\\n \\\"subActivityId\\\": \\\"47228ea1-a92b-46c8-8e17-cb952ef7f8d3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"87aae106-3cd3-4084-a623-9299e4fdb101\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74 ARID=354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.Http.CallContext/354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.ExecuteQuery/87aae106-3cd3-4084-a623-9299e4fdb101 \\u003e DN.FE.ExecuteQuery/47228ea1-a92b-46c8-8e17-cb952ef7f8d3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[04:25:24.699]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T04:25:20.0719077Z);\\nlet _endTime = datetime(2026-08-25T04:25:20.0719082Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bm8wcj6bwd2b'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T04:25:24.3156878Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74\\\",\\r\\n \\\"activityId\\\": \\\"354f4d7a-e0a3-4245-aead-8d5376f818a3\\\",\\r\\n \\\"subActivityId\\\": \\\"47228ea1-a92b-46c8-8e17-cb952ef7f8d3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"87aae106-3cd3-4084-a623-9299e4fdb101\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d5f87f2-9259-48f2-94c1-ca20f51aec74 ARID=354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.Http.CallContext/354f4d7a-e0a3-4245-aead-8d5376f818a3 \\u003e GW.ExecuteQuery/87aae106-3cd3-4084-a623-9299e4fdb101 \\u003e DN.FE.ExecuteQuery/47228ea1-a92b-46c8-8e17-cb952ef7f8d3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6666,"pr_state":"closed","pr_sha":"a86004571d5c4fc79ff255fd53d013393120edbf","final_merged_sha":"a86004571d5c4fc79ff255fd53d013393120edbf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T02:21:30Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"10665c3c849b8c7ece49b0dc1a98525113e3a1109a95138c29488b4338cce673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"e607efd27bdf5f0f02b0b2627c8fcd0b747075950cb39ab6c422f0ffaddbcc9f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:137]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00152ad60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-rlhmvt-xv6clc\" in resourcegroup=\"rg-np-version-upgrade-rlhmvt-dxdbz4whmbf6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-rlhmvt-dxdbz4whmbf6/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-rlhmvt-xv6clc/operationStatuses/08584139797153791389\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-u3utnosfhzsyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:137]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00152ad60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-rlhmvt-xv6clc\" in resourcegroup=\"rg-np-version-upgrade-rlhmvt-dxdbz4whmbf6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-rlhmvt-dxdbz4whmbf6/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-rlhmvt-xv6clc/operationStatuses/08584139797153791389\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-u3utnosfhzsyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5212740c2ffb00cb06412c5bcca2f933962b78eac0e75d815040a2654df8918a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"5c83e4b12f76857c14b38ba7e2400380e5142f913e512f879520f40194148815","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:137]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-wtgj75\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00152e100\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf/operationStatuses/08584139797161101429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-vcuqi2ogq2kwk' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:137]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-wtgj75\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00152e100\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf/operationStatuses/08584139797161101429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-vcuqi2ogq2kwk' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"68710b55179c18ca8dad283be47e26870934ca8e079472cbfdcc0a8adc36b37b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"af625ee5896ee6ce49c9e654442a846928a680f63a96ce2b39b4c1040f2836c7","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-listd9kcp5-zjrzlw\" in resourcegroup=\"ea-list-db5549k8k7r7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-db5549k8k7r7/providers/Microsoft.Resources/deployments/customer-infra-ea-listd9kcp5-zjrzlw/operationStatuses/08584139797158020904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-yygnj2pbjetam' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-listd9kcp5-zjrzlw\" in resourcegroup=\"ea-list-db5549k8k7r7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-db5549k8k7r7/providers/Microsoft.Resources/deployments/customer-infra-ea-listd9kcp5-zjrzlw/operationStatuses/08584139797158020904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-yygnj2pbjetam' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6a2f12e19ace74a26715c875336d2f024f83186fe167579eef75e7184a01ac25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"b2816cbf83c46fd984a6ad136b3252029110012c9e6f9c0547ab7a88bcac4f9a","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:407]: failed to create customer resources for boot diagnostics cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00124a120\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-sre-hcp-cluster-bootdiag-wshfnj\" in resourcegroup=\"admin-api-serialconsole-bootdiag-vlm24jhwfxfh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-serialconsole-bootdiag-vlm24jhwfxfh/providers/Microsoft.Resources/deployments/customer-infra-sre-hcp-cluster-bootdiag-wshfnj/operationStatuses/08584139797170845816\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-w3yf2qmes5oby' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:407]: failed to create customer resources for boot diagnostics cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00124a120\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-sre-hcp-cluster-bootdiag-wshfnj\" in resourcegroup=\"admin-api-serialconsole-bootdiag-vlm24jhwfxfh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-serialconsole-bootdiag-vlm24jhwfxfh/providers/Microsoft.Resources/deployments/customer-infra-sre-hcp-cluster-bootdiag-wshfnj/operationStatuses/08584139797170845816\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-w3yf2qmes5oby' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7a18222a02106be04e7fbc10bf299c01bba22a35416b9adb0b42cdacab749c55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"94154d206722ce8b1453f60448361cd8570bf51c717a7b40fa0937eebc3c9bdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"eef4016eff4f328bc4e6b8d0c0a4efcaa132ceba7408e01c03a4f07e211176c2","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d8c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-pullsecret-hcp-cluster-q9hglk\" in resourcegroup=\"pullsecret-test-5kr2hsv6hcfr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/pullsecret-test-5kr2hsv6hcfr/providers/Microsoft.Resources/deployments/customer-infra-pullsecret-hcp-cluster-q9hglk/operationStatuses/08584139797167198377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-je4kwf4e6h44k' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d8c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-pullsecret-hcp-cluster-q9hglk\" in resourcegroup=\"pullsecret-test-5kr2hsv6hcfr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/pullsecret-test-5kr2hsv6hcfr/providers/Microsoft.Resources/deployments/customer-infra-pullsecret-hcp-cluster-q9hglk/operationStatuses/08584139797167198377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-je4kwf4e6h44k' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"acd341d78791674e4da2c60183bddadf69f9488792ad78fcd8f38db6686027db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"alert","signature_id":"04107c06cdc1c97e2051bd30a01f9feaff1579d56b011fda20b5f0cc7435b337","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T03:20:32Z\n Ended: 2026-08-25T03:39:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4913920-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdbhnin5nlje0kom9tdeu4uptgmg6tm-u0k3n4i5m6h1a0q\", pod=\"router-559bc78df9-qwb7n\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sdbhnin5nlje0kom9tdeu4uptgmg6tm-u0k3n4i5m6h1a0q/router-559bc78df9-qwb7n has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"bbe2c52da6bbea75b392096f3ab9bd9950d014e82645151db30f389f5a6223cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"7f995c509ce4acadbbdef98d81915a02d692abceffa2a81b563b98b98cf85466","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rq5v9mrlc9n8/cluster-hs-pre-6fnkjh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7ea60\u003e: \n failed to create HCP cluster cluster-hs-pre-6fnkjh: failed waiting for cluster=\"cluster-hs-pre-6fnkjh\" in resourcegroup=\"rg-hs-pre--rq5v9mrlc9n8\" to finish creating: GET https://rp.j4913920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d26e415-5eaa-496f-8467-9de6cbee0e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d26e415-5eaa-496f-8467-9de6cbee0e6f\",\n \"name\": \"4d26e415-5eaa-496f-8467-9de6cbee0e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T03:07:08.995601074Z\",\n \"endTime\": \"2026-08-25T03:26:17.111880002Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rq5v9mrlc9n8/cluster-hs-pre-6fnkjh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7ea60\u003e: \n failed to create HCP cluster cluster-hs-pre-6fnkjh: failed waiting for cluster=\"cluster-hs-pre-6fnkjh\" in resourcegroup=\"rg-hs-pre--rq5v9mrlc9n8\" to finish creating: GET https://rp.j4913920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d26e415-5eaa-496f-8467-9de6cbee0e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d26e415-5eaa-496f-8467-9de6cbee0e6f\",\n \"name\": \"4d26e415-5eaa-496f-8467-9de6cbee0e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T03:07:08.995601074Z\",\n \"endTime\": \"2026-08-25T03:26:17.111880002Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c7da633061b0132f6185dbcbea78fd09dddfc995750f97c3d2967d208c45e839","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","failed_at":"e2e","signature_id":"6798ec01d31b1a13a551f0d58e2e35ad1b3449b1679cb8d8c473dee35aed6098","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000839700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-qdpn79\" in resourcegroup=\"rg-np-labels-taints-5t4klzqbfs75\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-5t4klzqbfs75/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-qdpn79/operationStatuses/08584139797158851285\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-eajlprfd56pyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000839700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-qdpn79\" in resourcegroup=\"rg-np-labels-taints-5t4klzqbfs75\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-5t4klzqbfs75/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-qdpn79/operationStatuses/08584139797158851285\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-eajlprfd56pyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"d5a9f482aee0f501c8ae8bda8cd06ffa03214aaf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T02:08:39Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"238ed17e7e994cd22f96c1fe31d02afe7d278d4727bfd95e253859647c783ad6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","occurred_at":"2026-08-25T02:08:39Z","failed_at":"alert","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T03:09:50Z\n Ended: 2026-08-25T03:32:50Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8366976-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"c0aca338d562157865b6ef1b6dce1d271f2c1ddf90124f14be81f4cb95946aab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","occurred_at":"2026-08-25T02:08:39Z","failed_at":"e2e","signature_id":"c9f1b4828c5cc783bc0db2584b06c52c8e5851d35045a6f094b2bdb76e6aa487","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0011bc0e0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[03:21:55.145]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:50.6783060Z);\\nlet _endTime = datetime(2026-08-25T03:21:50.6783064Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:55.1228509Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80\\\",\\r\\n \\\"activityId\\\": \\\"a533b28a-fda6-4117-9266-cedeb23e7dcb\\\",\\r\\n \\\"subActivityId\\\": \\\"9af34922-404d-4aac-a338-76029942029a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"717b06d7-8fad-4d02-95bc-b64968fec11b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80 ARID=a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.Http.CallContext/a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.ExecuteQuery/717b06d7-8fad-4d02-95bc-b64968fec11b \\u003e DN.FE.ExecuteQuery/9af34922-404d-4aac-a338-76029942029a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:21:55.223]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:50.6783060Z);\\nlet _endTime = datetime(2026-08-25T03:21:50.6783064Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:55.1228509Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80\\\",\\r\\n \\\"activityId\\\": \\\"a533b28a-fda6-4117-9266-cedeb23e7dcb\\\",\\r\\n \\\"subActivityId\\\": \\\"9af34922-404d-4aac-a338-76029942029a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"717b06d7-8fad-4d02-95bc-b64968fec11b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80 ARID=a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.Http.CallContext/a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.ExecuteQuery/717b06d7-8fad-4d02-95bc-b64968fec11b \\u003e DN.FE.ExecuteQuery/9af34922-404d-4aac-a338-76029942029a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[03:21:59.054]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:55.2959172Z);\\nlet _endTime = datetime(2026-08-25T03:21:55.2959176Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:59.0220720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a\\\",\\r\\n \\\"activityId\\\": \\\"30c06b36-6f95-41ea-b07a-99c2df6725ac\\\",\\r\\n \\\"subActivityId\\\": \\\"f0448801-56e1-4549-948a-2ee0610e96e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e176f93c-c918-49b7-a0cc-4942b4706d95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a ARID=30c06b36-6f95-41ea-b07a-99c2df6725ac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e176f93c-c918-49b7-a0cc-4942b4706d95 \\u003e DN.FE.ExecuteQuery/f0448801-56e1-4549-948a-2ee0610e96e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:21:59.140]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:55.2959172Z);\\nlet _endTime = datetime(2026-08-25T03:21:55.2959176Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:59.0220720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a\\\",\\r\\n \\\"activityId\\\": \\\"30c06b36-6f95-41ea-b07a-99c2df6725ac\\\",\\r\\n \\\"subActivityId\\\": \\\"f0448801-56e1-4549-948a-2ee0610e96e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e176f93c-c918-49b7-a0cc-4942b4706d95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a ARID=30c06b36-6f95-41ea-b07a-99c2df6725ac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e176f93c-c918-49b7-a0cc-4942b4706d95 \\u003e DN.FE.ExecuteQuery/f0448801-56e1-4549-948a-2ee0610e96e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[03:22:08.110]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:22:03.5199134Z);\\nlet _endTime = datetime(2026-08-25T03:22:03.5199138Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:22:08.0879976Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb\\\",\\r\\n \\\"activityId\\\": \\\"a25a050e-8242-4a91-95a9-86fa711bcc5a\\\",\\r\\n \\\"subActivityId\\\": \\\"8e9b47fe-6372-4513-bf48-f4ed0d67e92b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cd9d751-12b3-44c4-9789-8e18a838b311\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb ARID=a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.Http.CallContext/a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.ExecuteQuery/8cd9d751-12b3-44c4-9789-8e18a838b311 \\u003e DN.FE.ExecuteQuery/8e9b47fe-6372-4513-bf48-f4ed0d67e92b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:22:08.617]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:22:03.5199134Z);\\nlet _endTime = datetime(2026-08-25T03:22:03.5199138Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:22:08.0879976Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb\\\",\\r\\n \\\"activityId\\\": \\\"a25a050e-8242-4a91-95a9-86fa711bcc5a\\\",\\r\\n \\\"subActivityId\\\": \\\"8e9b47fe-6372-4513-bf48-f4ed0d67e92b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cd9d751-12b3-44c4-9789-8e18a838b311\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb ARID=a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.Http.CallContext/a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.ExecuteQuery/8cd9d751-12b3-44c4-9789-8e18a838b311 \\u003e DN.FE.ExecuteQuery/8e9b47fe-6372-4513-bf48-f4ed0d67e92b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0011bc0e0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[03:21:55.145]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:50.6783060Z);\\nlet _endTime = datetime(2026-08-25T03:21:50.6783064Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:55.1228509Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80\\\",\\r\\n \\\"activityId\\\": \\\"a533b28a-fda6-4117-9266-cedeb23e7dcb\\\",\\r\\n \\\"subActivityId\\\": \\\"9af34922-404d-4aac-a338-76029942029a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"717b06d7-8fad-4d02-95bc-b64968fec11b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80 ARID=a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.Http.CallContext/a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.ExecuteQuery/717b06d7-8fad-4d02-95bc-b64968fec11b \\u003e DN.FE.ExecuteQuery/9af34922-404d-4aac-a338-76029942029a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:21:55.223]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:50.6783060Z);\\nlet _endTime = datetime(2026-08-25T03:21:50.6783064Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:55.1228509Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80\\\",\\r\\n \\\"activityId\\\": \\\"a533b28a-fda6-4117-9266-cedeb23e7dcb\\\",\\r\\n \\\"subActivityId\\\": \\\"9af34922-404d-4aac-a338-76029942029a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"717b06d7-8fad-4d02-95bc-b64968fec11b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;1dd61c9f-5e11-4510-b677-6a52785b7a80 ARID=a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.Http.CallContext/a533b28a-fda6-4117-9266-cedeb23e7dcb \\u003e GW.ExecuteQuery/717b06d7-8fad-4d02-95bc-b64968fec11b \\u003e DN.FE.ExecuteQuery/9af34922-404d-4aac-a338-76029942029a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[03:21:59.054]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:55.2959172Z);\\nlet _endTime = datetime(2026-08-25T03:21:55.2959176Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:59.0220720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a\\\",\\r\\n \\\"activityId\\\": \\\"30c06b36-6f95-41ea-b07a-99c2df6725ac\\\",\\r\\n \\\"subActivityId\\\": \\\"f0448801-56e1-4549-948a-2ee0610e96e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e176f93c-c918-49b7-a0cc-4942b4706d95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a ARID=30c06b36-6f95-41ea-b07a-99c2df6725ac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e176f93c-c918-49b7-a0cc-4942b4706d95 \\u003e DN.FE.ExecuteQuery/f0448801-56e1-4549-948a-2ee0610e96e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:21:59.140]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:21:55.2959172Z);\\nlet _endTime = datetime(2026-08-25T03:21:55.2959176Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:21:59.0220720Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a\\\",\\r\\n \\\"activityId\\\": \\\"30c06b36-6f95-41ea-b07a-99c2df6725ac\\\",\\r\\n \\\"subActivityId\\\": \\\"f0448801-56e1-4549-948a-2ee0610e96e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e176f93c-c918-49b7-a0cc-4942b4706d95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;70efb98e-3fd3-4ed1-8a8f-46228dd97b8a ARID=30c06b36-6f95-41ea-b07a-99c2df6725ac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/71bb00b7-67f3-4178-9894-a7f1ac28d455 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e176f93c-c918-49b7-a0cc-4942b4706d95 \\u003e DN.FE.ExecuteQuery/f0448801-56e1-4549-948a-2ee0610e96e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[03:22:08.110]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:22:03.5199134Z);\\nlet _endTime = datetime(2026-08-25T03:22:03.5199138Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:22:08.0879976Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb\\\",\\r\\n \\\"activityId\\\": \\\"a25a050e-8242-4a91-95a9-86fa711bcc5a\\\",\\r\\n \\\"subActivityId\\\": \\\"8e9b47fe-6372-4513-bf48-f4ed0d67e92b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cd9d751-12b3-44c4-9789-8e18a838b311\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb ARID=a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.Http.CallContext/a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.ExecuteQuery/8cd9d751-12b3-44c4-9789-8e18a838b311 \\u003e DN.FE.ExecuteQuery/8e9b47fe-6372-4513-bf48-f4ed0d67e92b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:22:08.617]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:22:03.5199134Z);\\nlet _endTime = datetime(2026-08-25T03:22:03.5199138Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9f2h9xs6hlg2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:22:08.0879976Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb\\\",\\r\\n \\\"activityId\\\": \\\"a25a050e-8242-4a91-95a9-86fa711bcc5a\\\",\\r\\n \\\"subActivityId\\\": \\\"8e9b47fe-6372-4513-bf48-f4ed0d67e92b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cd9d751-12b3-44c4-9789-8e18a838b311\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d2623593-01a6-417f-bb81-7935f556c2fb ARID=a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.Http.CallContext/a25a050e-8242-4a91-95a9-86fa711bcc5a \\u003e GW.ExecuteQuery/8cd9d751-12b3-44c4-9789-8e18a838b311 \\u003e DN.FE.ExecuteQuery/8e9b47fe-6372-4513-bf48-f4ed0d67e92b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"86bc57f75bd4bc7d3ab9f59b27225fa5422625d2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T02:08:38Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"896f159b9a530657cedf6b4f3a9c19a5c51dfbf0175fa56abc06076683a64915","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","occurred_at":"2026-08-25T02:08:38Z","failed_at":"e2e","signature_id":"28297427d9d2bd8653dc649081b99f9573b907029f55909add770c57baa41099","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00059c490\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[03:18:05.024]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:01.4708744Z);\\nlet _endTime = datetime(2026-08-25T03:18:01.4708751Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:04.8684708Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43\\\",\\r\\n \\\"activityId\\\": \\\"df7eb063-da18-4ce9-b194-404eed18bc06\\\",\\r\\n \\\"subActivityId\\\": \\\"ba927633-d0c2-42e7-92bb-ec25d786d33b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7262baa4-5b98-4685-985c-814b985c2fe1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43 ARID=df7eb063-da18-4ce9-b194-404eed18bc06 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/7262baa4-5b98-4685-985c-814b985c2fe1 \\u003e DN.FE.ExecuteQuery/ba927633-d0c2-42e7-92bb-ec25d786d33b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:18:05.528]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:01.4708744Z);\\nlet _endTime = datetime(2026-08-25T03:18:01.4708751Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:04.8684708Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43\\\",\\r\\n \\\"activityId\\\": \\\"df7eb063-da18-4ce9-b194-404eed18bc06\\\",\\r\\n \\\"subActivityId\\\": \\\"ba927633-d0c2-42e7-92bb-ec25d786d33b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7262baa4-5b98-4685-985c-814b985c2fe1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43 ARID=df7eb063-da18-4ce9-b194-404eed18bc06 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/7262baa4-5b98-4685-985c-814b985c2fe1 \\u003e DN.FE.ExecuteQuery/ba927633-d0c2-42e7-92bb-ec25d786d33b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[03:18:08.793]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:05.5866186Z);\\nlet _endTime = datetime(2026-08-25T03:18:05.5866195Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:08.7716035Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2\\\",\\r\\n \\\"activityId\\\": \\\"1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c\\\",\\r\\n \\\"subActivityId\\\": \\\"a66f847d-efa4-4dbd-9d36-35f6dcf8cac6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"17563e2f-0c3a-4f73-a2a6-b81ec16b170d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2 ARID=1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.Http.CallContext/1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.ExecuteQuery/17563e2f-0c3a-4f73-a2a6-b81ec16b170d \\u003e DN.FE.ExecuteQuery/a66f847d-efa4-4dbd-9d36-35f6dcf8cac6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:18:08.843]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:05.5866186Z);\\nlet _endTime = datetime(2026-08-25T03:18:05.5866195Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:08.7716035Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2\\\",\\r\\n \\\"activityId\\\": \\\"1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c\\\",\\r\\n \\\"subActivityId\\\": \\\"a66f847d-efa4-4dbd-9d36-35f6dcf8cac6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"17563e2f-0c3a-4f73-a2a6-b81ec16b170d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2 ARID=1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.Http.CallContext/1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.ExecuteQuery/17563e2f-0c3a-4f73-a2a6-b81ec16b170d \\u003e DN.FE.ExecuteQuery/a66f847d-efa4-4dbd-9d36-35f6dcf8cac6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[03:18:14.785]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:11.8591619Z);\\nlet _endTime = datetime(2026-08-25T03:18:11.8591624Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:14.7664174Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3\\\",\\r\\n \\\"activityId\\\": \\\"1ab49792-00ec-450e-8935-2f8d1308a67b\\\",\\r\\n \\\"subActivityId\\\": \\\"cd0112e8-bbfc-4f60-bbba-6257d1fca5d0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3 ARID=1ab49792-00ec-450e-8935-2f8d1308a67b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f \\u003e DN.FE.ExecuteQuery/cd0112e8-bbfc-4f60-bbba-6257d1fca5d0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:18:14.879]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:11.8591619Z);\\nlet _endTime = datetime(2026-08-25T03:18:11.8591624Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:14.7664174Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3\\\",\\r\\n \\\"activityId\\\": \\\"1ab49792-00ec-450e-8935-2f8d1308a67b\\\",\\r\\n \\\"subActivityId\\\": \\\"cd0112e8-bbfc-4f60-bbba-6257d1fca5d0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3 ARID=1ab49792-00ec-450e-8935-2f8d1308a67b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f \\u003e DN.FE.ExecuteQuery/cd0112e8-bbfc-4f60-bbba-6257d1fca5d0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00059c490\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[03:18:05.024]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:01.4708744Z);\\nlet _endTime = datetime(2026-08-25T03:18:01.4708751Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:04.8684708Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43\\\",\\r\\n \\\"activityId\\\": \\\"df7eb063-da18-4ce9-b194-404eed18bc06\\\",\\r\\n \\\"subActivityId\\\": \\\"ba927633-d0c2-42e7-92bb-ec25d786d33b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7262baa4-5b98-4685-985c-814b985c2fe1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43 ARID=df7eb063-da18-4ce9-b194-404eed18bc06 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/7262baa4-5b98-4685-985c-814b985c2fe1 \\u003e DN.FE.ExecuteQuery/ba927633-d0c2-42e7-92bb-ec25d786d33b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:18:05.528]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:01.4708744Z);\\nlet _endTime = datetime(2026-08-25T03:18:01.4708751Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:04.8684708Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43\\\",\\r\\n \\\"activityId\\\": \\\"df7eb063-da18-4ce9-b194-404eed18bc06\\\",\\r\\n \\\"subActivityId\\\": \\\"ba927633-d0c2-42e7-92bb-ec25d786d33b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7262baa4-5b98-4685-985c-814b985c2fe1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;67bf74fc-55e4-44fe-b1ce-3354120e7c43 ARID=df7eb063-da18-4ce9-b194-404eed18bc06 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/380c0153-ebcd-4bae-9813-2119b7da3734 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/7262baa4-5b98-4685-985c-814b985c2fe1 \\u003e DN.FE.ExecuteQuery/ba927633-d0c2-42e7-92bb-ec25d786d33b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[03:18:08.793]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:05.5866186Z);\\nlet _endTime = datetime(2026-08-25T03:18:05.5866195Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:08.7716035Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2\\\",\\r\\n \\\"activityId\\\": \\\"1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c\\\",\\r\\n \\\"subActivityId\\\": \\\"a66f847d-efa4-4dbd-9d36-35f6dcf8cac6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"17563e2f-0c3a-4f73-a2a6-b81ec16b170d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2 ARID=1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.Http.CallContext/1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.ExecuteQuery/17563e2f-0c3a-4f73-a2a6-b81ec16b170d \\u003e DN.FE.ExecuteQuery/a66f847d-efa4-4dbd-9d36-35f6dcf8cac6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:18:08.843]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:05.5866186Z);\\nlet _endTime = datetime(2026-08-25T03:18:05.5866195Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:08.7716035Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2\\\",\\r\\n \\\"activityId\\\": \\\"1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c\\\",\\r\\n \\\"subActivityId\\\": \\\"a66f847d-efa4-4dbd-9d36-35f6dcf8cac6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"17563e2f-0c3a-4f73-a2a6-b81ec16b170d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e8bce747-327d-497b-8794-664b10bd36f2 ARID=1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.Http.CallContext/1fbcf01f-9b69-4f3c-8c1a-a38dd75ca84c \\u003e GW.ExecuteQuery/17563e2f-0c3a-4f73-a2a6-b81ec16b170d \\u003e DN.FE.ExecuteQuery/a66f847d-efa4-4dbd-9d36-35f6dcf8cac6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[03:18:14.785]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:11.8591619Z);\\nlet _endTime = datetime(2026-08-25T03:18:11.8591624Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:14.7664174Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3\\\",\\r\\n \\\"activityId\\\": \\\"1ab49792-00ec-450e-8935-2f8d1308a67b\\\",\\r\\n \\\"subActivityId\\\": \\\"cd0112e8-bbfc-4f60-bbba-6257d1fca5d0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3 ARID=1ab49792-00ec-450e-8935-2f8d1308a67b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f \\u003e DN.FE.ExecuteQuery/cd0112e8-bbfc-4f60-bbba-6257d1fca5d0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[03:18:14.879]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T03:18:11.8591619Z);\\nlet _endTime = datetime(2026-08-25T03:18:11.8591624Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lm44zbt5pvpw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T03:18:14.7664174Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3\\\",\\r\\n \\\"activityId\\\": \\\"1ab49792-00ec-450e-8935-2f8d1308a67b\\\",\\r\\n \\\"subActivityId\\\": \\\"cd0112e8-bbfc-4f60-bbba-6257d1fca5d0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;18680eb2-412c-4a27-a7da-c1683ef1b7d3 ARID=1ab49792-00ec-450e-8935-2f8d1308a67b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/09280cdc-64c4-4ec5-8daa-c623f8d1377e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/a0bd9d76-07f5-4c1e-88af-6e3ce855fa1f \\u003e DN.FE.ExecuteQuery/cd0112e8-bbfc-4f60-bbba-6257d1fca5d0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"cc9576f73a0db2cc26ca6de7a00964e1c4fe8d494817e48328b7dc77f4e3457d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","occurred_at":"2026-08-25T02:08:38Z","failed_at":"e2e","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066535932694528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T01:48:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a0a0df6d0eddfb29188563e2b186a886e42bce0f8e355ea234cfcb9d93c89ede","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066535932694528","occurred_at":"2026-08-25T01:48:25Z","failed_at":"e2e","signature_id":"002c1d2e05187089a12c62fceba5d319b6b2b88aa8f59eba28a6c84b20651f56","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b44150\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[02:57:13.167]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:10.1617793Z);\\nlet _endTime = datetime(2026-08-25T02:57:10.1617797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:13.1478158Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7\\\",\\r\\n \\\"activityId\\\": \\\"abc94df6-4dd7-47ea-8276-cda1a81bec1a\\\",\\r\\n \\\"subActivityId\\\": \\\"98b66884-faac-4b45-aa7f-137af0a5c09d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"62d02ff5-f34f-441b-90f9-a785f38b3d63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7 ARID=abc94df6-4dd7-47ea-8276-cda1a81bec1a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/62d02ff5-f34f-441b-90f9-a785f38b3d63 \\u003e DN.FE.ExecuteQuery/98b66884-faac-4b45-aa7f-137af0a5c09d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:57:13.217]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:10.1617793Z);\\nlet _endTime = datetime(2026-08-25T02:57:10.1617797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:13.1478158Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7\\\",\\r\\n \\\"activityId\\\": \\\"abc94df6-4dd7-47ea-8276-cda1a81bec1a\\\",\\r\\n \\\"subActivityId\\\": \\\"98b66884-faac-4b45-aa7f-137af0a5c09d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"62d02ff5-f34f-441b-90f9-a785f38b3d63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7 ARID=abc94df6-4dd7-47ea-8276-cda1a81bec1a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/62d02ff5-f34f-441b-90f9-a785f38b3d63 \\u003e DN.FE.ExecuteQuery/98b66884-faac-4b45-aa7f-137af0a5c09d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[02:57:16.056]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:13.2830937Z);\\nlet _endTime = datetime(2026-08-25T02:57:13.2830941Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:16.0360865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254\\\",\\r\\n \\\"activityId\\\": \\\"1f294821-56f3-486c-844a-fe730b1f9bd6\\\",\\r\\n \\\"subActivityId\\\": \\\"fd707038-0134-45f5-9cfe-c1fc2424f14c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5d8c0304-a8ff-4084-ad97-bb51505ffc4f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254 ARID=1f294821-56f3-486c-844a-fe730b1f9bd6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5d8c0304-a8ff-4084-ad97-bb51505ffc4f \\u003e DN.FE.ExecuteQuery/fd707038-0134-45f5-9cfe-c1fc2424f14c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:57:16.106]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:13.2830937Z);\\nlet _endTime = datetime(2026-08-25T02:57:13.2830941Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:16.0360865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254\\\",\\r\\n \\\"activityId\\\": \\\"1f294821-56f3-486c-844a-fe730b1f9bd6\\\",\\r\\n \\\"subActivityId\\\": \\\"fd707038-0134-45f5-9cfe-c1fc2424f14c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5d8c0304-a8ff-4084-ad97-bb51505ffc4f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254 ARID=1f294821-56f3-486c-844a-fe730b1f9bd6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5d8c0304-a8ff-4084-ad97-bb51505ffc4f \\u003e DN.FE.ExecuteQuery/fd707038-0134-45f5-9cfe-c1fc2424f14c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[02:57:21.868]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:19.1309905Z);\\nlet _endTime = datetime(2026-08-25T02:57:19.1309912Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:21.8451692Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb\\\",\\r\\n \\\"activityId\\\": \\\"d2ea1670-69d4-4889-a9c3-a16935ec60b7\\\",\\r\\n \\\"subActivityId\\\": \\\"413c5e56-d83d-4be8-92fc-a41e2c3ddd3b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d2e955bc-d7f9-4c57-96df-785494e9e685\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb ARID=d2ea1670-69d4-4889-a9c3-a16935ec60b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d2e955bc-d7f9-4c57-96df-785494e9e685 \\u003e DN.FE.ExecuteQuery/413c5e56-d83d-4be8-92fc-a41e2c3ddd3b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:57:21.972]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:19.1309905Z);\\nlet _endTime = datetime(2026-08-25T02:57:19.1309912Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:21.8451692Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb\\\",\\r\\n \\\"activityId\\\": \\\"d2ea1670-69d4-4889-a9c3-a16935ec60b7\\\",\\r\\n \\\"subActivityId\\\": \\\"413c5e56-d83d-4be8-92fc-a41e2c3ddd3b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d2e955bc-d7f9-4c57-96df-785494e9e685\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb ARID=d2ea1670-69d4-4889-a9c3-a16935ec60b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d2e955bc-d7f9-4c57-96df-785494e9e685 \\u003e DN.FE.ExecuteQuery/413c5e56-d83d-4be8-92fc-a41e2c3ddd3b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b44150\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[02:57:13.167]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:10.1617793Z);\\nlet _endTime = datetime(2026-08-25T02:57:10.1617797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:13.1478158Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7\\\",\\r\\n \\\"activityId\\\": \\\"abc94df6-4dd7-47ea-8276-cda1a81bec1a\\\",\\r\\n \\\"subActivityId\\\": \\\"98b66884-faac-4b45-aa7f-137af0a5c09d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"62d02ff5-f34f-441b-90f9-a785f38b3d63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7 ARID=abc94df6-4dd7-47ea-8276-cda1a81bec1a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/62d02ff5-f34f-441b-90f9-a785f38b3d63 \\u003e DN.FE.ExecuteQuery/98b66884-faac-4b45-aa7f-137af0a5c09d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:57:13.217]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:10.1617793Z);\\nlet _endTime = datetime(2026-08-25T02:57:10.1617797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:13.1478158Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7\\\",\\r\\n \\\"activityId\\\": \\\"abc94df6-4dd7-47ea-8276-cda1a81bec1a\\\",\\r\\n \\\"subActivityId\\\": \\\"98b66884-faac-4b45-aa7f-137af0a5c09d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"62d02ff5-f34f-441b-90f9-a785f38b3d63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;76052f65-2aef-419e-8fea-b6379e3d72a7 ARID=abc94df6-4dd7-47ea-8276-cda1a81bec1a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ca624d7f-ad79-47bc-8157-638140dc10be \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/62d02ff5-f34f-441b-90f9-a785f38b3d63 \\u003e DN.FE.ExecuteQuery/98b66884-faac-4b45-aa7f-137af0a5c09d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[02:57:16.056]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:13.2830937Z);\\nlet _endTime = datetime(2026-08-25T02:57:13.2830941Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:16.0360865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254\\\",\\r\\n \\\"activityId\\\": \\\"1f294821-56f3-486c-844a-fe730b1f9bd6\\\",\\r\\n \\\"subActivityId\\\": \\\"fd707038-0134-45f5-9cfe-c1fc2424f14c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5d8c0304-a8ff-4084-ad97-bb51505ffc4f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254 ARID=1f294821-56f3-486c-844a-fe730b1f9bd6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5d8c0304-a8ff-4084-ad97-bb51505ffc4f \\u003e DN.FE.ExecuteQuery/fd707038-0134-45f5-9cfe-c1fc2424f14c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:57:16.106]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:13.2830937Z);\\nlet _endTime = datetime(2026-08-25T02:57:13.2830941Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:16.0360865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254\\\",\\r\\n \\\"activityId\\\": \\\"1f294821-56f3-486c-844a-fe730b1f9bd6\\\",\\r\\n \\\"subActivityId\\\": \\\"fd707038-0134-45f5-9cfe-c1fc2424f14c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5d8c0304-a8ff-4084-ad97-bb51505ffc4f\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5ef0c71c-0b3a-49ca-95fd-0e58a9d1c254 ARID=1f294821-56f3-486c-844a-fe730b1f9bd6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b75f998b-4331-452f-bc8c-8eef2eddc571 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5d8c0304-a8ff-4084-ad97-bb51505ffc4f \\u003e DN.FE.ExecuteQuery/fd707038-0134-45f5-9cfe-c1fc2424f14c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[02:57:21.868]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:19.1309905Z);\\nlet _endTime = datetime(2026-08-25T02:57:19.1309912Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:21.8451692Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb\\\",\\r\\n \\\"activityId\\\": \\\"d2ea1670-69d4-4889-a9c3-a16935ec60b7\\\",\\r\\n \\\"subActivityId\\\": \\\"413c5e56-d83d-4be8-92fc-a41e2c3ddd3b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d2e955bc-d7f9-4c57-96df-785494e9e685\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb ARID=d2ea1670-69d4-4889-a9c3-a16935ec60b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d2e955bc-d7f9-4c57-96df-785494e9e685 \\u003e DN.FE.ExecuteQuery/413c5e56-d83d-4be8-92fc-a41e2c3ddd3b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:57:21.972]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:57:19.1309905Z);\\nlet _endTime = datetime(2026-08-25T02:57:19.1309912Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-x67lfjm8dbhl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:57:21.8451692Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb\\\",\\r\\n \\\"activityId\\\": \\\"d2ea1670-69d4-4889-a9c3-a16935ec60b7\\\",\\r\\n \\\"subActivityId\\\": \\\"413c5e56-d83d-4be8-92fc-a41e2c3ddd3b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d2e955bc-d7f9-4c57-96df-785494e9e685\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e9970c55-d97f-4a19-b966-86c28211c9bb ARID=d2ea1670-69d4-4889-a9c3-a16935ec60b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f00c2e84-e470-4abc-ba5b-53d753930ad7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d2e955bc-d7f9-4c57-96df-785494e9e685 \\u003e DN.FE.ExecuteQuery/413c5e56-d83d-4be8-92fc-a41e2c3ddd3b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066407356305408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6554,"pr_state":"open","pr_sha":"4e67bc77de1a80644494c4c53d6416cbb8167300","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T01:47:54Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f7581c4db592f168341e97d7c9c51ed6bd0ba3cf53eea79a091010940381a6aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066407356305408","occurred_at":"2026-08-25T01:47:54Z","failed_at":"e2e","signature_id":"805bce7e0aab4147685c771f0375ca865818fb3186f1e427a03da21aa4561fc2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b8a1a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[02:46:49.041]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:44.9847112Z);\\nlet _endTime = datetime(2026-08-25T02:46:44.9847117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:49.0230725Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df\\\",\\r\\n \\\"activityId\\\": \\\"10d87903-bbdc-49d7-b182-c7bd286e370d\\\",\\r\\n \\\"subActivityId\\\": \\\"5ed51d3f-0d4a-49f1-aac5-f03667ec93a0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df ARID=10d87903-bbdc-49d7-b182-c7bd286e370d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e \\u003e DN.FE.ExecuteQuery/5ed51d3f-0d4a-49f1-aac5-f03667ec93a0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:46:49.088]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:44.9847112Z);\\nlet _endTime = datetime(2026-08-25T02:46:44.9847117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:49.0230725Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df\\\",\\r\\n \\\"activityId\\\": \\\"10d87903-bbdc-49d7-b182-c7bd286e370d\\\",\\r\\n \\\"subActivityId\\\": \\\"5ed51d3f-0d4a-49f1-aac5-f03667ec93a0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df ARID=10d87903-bbdc-49d7-b182-c7bd286e370d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e \\u003e DN.FE.ExecuteQuery/5ed51d3f-0d4a-49f1-aac5-f03667ec93a0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[02:46:52.156]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:49.1536737Z);\\nlet _endTime = datetime(2026-08-25T02:46:49.1536742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:52.1327710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c\\\",\\r\\n \\\"activityId\\\": \\\"66809e6d-c584-4f5e-b6ed-0c23de2db572\\\",\\r\\n \\\"subActivityId\\\": \\\"47e81ccf-6d17-42f1-a388-36872a1b72f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"09b71b7c-7e90-4aec-ad2b-df776045d2a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c ARID=66809e6d-c584-4f5e-b6ed-0c23de2db572 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/09b71b7c-7e90-4aec-ad2b-df776045d2a3 \\u003e DN.FE.ExecuteQuery/47e81ccf-6d17-42f1-a388-36872a1b72f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:46:52.208]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:49.1536737Z);\\nlet _endTime = datetime(2026-08-25T02:46:49.1536742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:52.1327710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c\\\",\\r\\n \\\"activityId\\\": \\\"66809e6d-c584-4f5e-b6ed-0c23de2db572\\\",\\r\\n \\\"subActivityId\\\": \\\"47e81ccf-6d17-42f1-a388-36872a1b72f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"09b71b7c-7e90-4aec-ad2b-df776045d2a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c ARID=66809e6d-c584-4f5e-b6ed-0c23de2db572 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/09b71b7c-7e90-4aec-ad2b-df776045d2a3 \\u003e DN.FE.ExecuteQuery/47e81ccf-6d17-42f1-a388-36872a1b72f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[02:46:58.829]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:55.2542072Z);\\nlet _endTime = datetime(2026-08-25T02:46:55.2542077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:58.8067947Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908\\\",\\r\\n \\\"activityId\\\": \\\"1a1d50bf-a610-43af-ae94-ff7858999b2c\\\",\\r\\n \\\"subActivityId\\\": \\\"e3430530-027f-4ee3-a20d-c3539326e24d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0afc02b0-9367-4ee2-a98c-86038a9bbbcc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908 ARID=1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.Http.CallContext/1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.ExecuteQuery/0afc02b0-9367-4ee2-a98c-86038a9bbbcc \\u003e DN.FE.ExecuteQuery/e3430530-027f-4ee3-a20d-c3539326e24d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:46:58.936]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:55.2542072Z);\\nlet _endTime = datetime(2026-08-25T02:46:55.2542077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:58.8067947Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908\\\",\\r\\n \\\"activityId\\\": \\\"1a1d50bf-a610-43af-ae94-ff7858999b2c\\\",\\r\\n \\\"subActivityId\\\": \\\"e3430530-027f-4ee3-a20d-c3539326e24d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0afc02b0-9367-4ee2-a98c-86038a9bbbcc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908 ARID=1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.Http.CallContext/1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.ExecuteQuery/0afc02b0-9367-4ee2-a98c-86038a9bbbcc \\u003e DN.FE.ExecuteQuery/e3430530-027f-4ee3-a20d-c3539326e24d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b8a1a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[02:46:49.041]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:44.9847112Z);\\nlet _endTime = datetime(2026-08-25T02:46:44.9847117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:49.0230725Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df\\\",\\r\\n \\\"activityId\\\": \\\"10d87903-bbdc-49d7-b182-c7bd286e370d\\\",\\r\\n \\\"subActivityId\\\": \\\"5ed51d3f-0d4a-49f1-aac5-f03667ec93a0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df ARID=10d87903-bbdc-49d7-b182-c7bd286e370d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e \\u003e DN.FE.ExecuteQuery/5ed51d3f-0d4a-49f1-aac5-f03667ec93a0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:46:49.088]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:44.9847112Z);\\nlet _endTime = datetime(2026-08-25T02:46:44.9847117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:49.0230725Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df\\\",\\r\\n \\\"activityId\\\": \\\"10d87903-bbdc-49d7-b182-c7bd286e370d\\\",\\r\\n \\\"subActivityId\\\": \\\"5ed51d3f-0d4a-49f1-aac5-f03667ec93a0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c670b93e-8900-4b77-a4d1-df5f562291df ARID=10d87903-bbdc-49d7-b182-c7bd286e370d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/ec65d60f-4372-4716-b968-285059eae484 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dbaf8df9-4a6c-4f51-926c-6a1d0b7a7a2e \\u003e DN.FE.ExecuteQuery/5ed51d3f-0d4a-49f1-aac5-f03667ec93a0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[02:46:52.156]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:49.1536737Z);\\nlet _endTime = datetime(2026-08-25T02:46:49.1536742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:52.1327710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c\\\",\\r\\n \\\"activityId\\\": \\\"66809e6d-c584-4f5e-b6ed-0c23de2db572\\\",\\r\\n \\\"subActivityId\\\": \\\"47e81ccf-6d17-42f1-a388-36872a1b72f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"09b71b7c-7e90-4aec-ad2b-df776045d2a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c ARID=66809e6d-c584-4f5e-b6ed-0c23de2db572 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/09b71b7c-7e90-4aec-ad2b-df776045d2a3 \\u003e DN.FE.ExecuteQuery/47e81ccf-6d17-42f1-a388-36872a1b72f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:46:52.208]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:49.1536737Z);\\nlet _endTime = datetime(2026-08-25T02:46:49.1536742Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:52.1327710Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c\\\",\\r\\n \\\"activityId\\\": \\\"66809e6d-c584-4f5e-b6ed-0c23de2db572\\\",\\r\\n \\\"subActivityId\\\": \\\"47e81ccf-6d17-42f1-a388-36872a1b72f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"09b71b7c-7e90-4aec-ad2b-df776045d2a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5cc7d5ee-afcd-41e8-afbe-777cb651042c ARID=66809e6d-c584-4f5e-b6ed-0c23de2db572 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/78cb5fbb-a4cc-461a-9fcc-8fc8bc845f2a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/09b71b7c-7e90-4aec-ad2b-df776045d2a3 \\u003e DN.FE.ExecuteQuery/47e81ccf-6d17-42f1-a388-36872a1b72f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[02:46:58.829]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:55.2542072Z);\\nlet _endTime = datetime(2026-08-25T02:46:55.2542077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:58.8067947Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908\\\",\\r\\n \\\"activityId\\\": \\\"1a1d50bf-a610-43af-ae94-ff7858999b2c\\\",\\r\\n \\\"subActivityId\\\": \\\"e3430530-027f-4ee3-a20d-c3539326e24d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0afc02b0-9367-4ee2-a98c-86038a9bbbcc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908 ARID=1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.Http.CallContext/1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.ExecuteQuery/0afc02b0-9367-4ee2-a98c-86038a9bbbcc \\u003e DN.FE.ExecuteQuery/e3430530-027f-4ee3-a20d-c3539326e24d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[02:46:58.936]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T02:46:55.2542072Z);\\nlet _endTime = datetime(2026-08-25T02:46:55.2542077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6dxkrlzwnlg5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T02:46:58.8067947Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908\\\",\\r\\n \\\"activityId\\\": \\\"1a1d50bf-a610-43af-ae94-ff7858999b2c\\\",\\r\\n \\\"subActivityId\\\": \\\"e3430530-027f-4ee3-a20d-c3539326e24d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0afc02b0-9367-4ee2-a98c-86038a9bbbcc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;edeaa02f-046d-43fc-b2ab-62ec451b6908 ARID=1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.Http.CallContext/1a1d50bf-a610-43af-ae94-ff7858999b2c \\u003e GW.ExecuteQuery/0afc02b0-9367-4ee2-a98c-86038a9bbbcc \\u003e DN.FE.ExecuteQuery/e3430530-027f-4ee3-a20d-c3539326e24d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6666,"pr_state":"closed","pr_sha":"a86004571d5c4fc79ff255fd53d013393120edbf","final_merged_sha":"a86004571d5c4fc79ff255fd53d013393120edbf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T00:21:02Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"21b93cd0dcc567842ee078f54d2e5312a5aeb291d26ce6f778f468fb9043c398","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","failed_at":"e2e","signature_id":"983555e8ae88df4ba006bdd3a98018885a3dc5949799b00108d57edd8321b6fb","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100c6c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7hms8hpjlmgg\" to finish creating: GET https://rp.j2511104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34872e1a-7b5d-4827-9041-710b4ce9f917\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34872e1a-7b5d-4827-9041-710b4ce9f917\",\n \"name\": \"34872e1a-7b5d-4827-9041-710b4ce9f917\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T01:11:23.297460639Z\",\n \"endTime\": \"2026-08-25T01:30:30.366070286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100c6c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7hms8hpjlmgg\" to finish creating: GET https://rp.j2511104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34872e1a-7b5d-4827-9041-710b4ce9f917\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34872e1a-7b5d-4827-9041-710b4ce9f917\",\n \"name\": \"34872e1a-7b5d-4827-9041-710b4ce9f917\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T01:11:23.297460639Z\",\n \"endTime\": \"2026-08-25T01:30:30.366070286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fda68253c5b31772800d3b0a92cf653d5bffa9689418d52ed09686cad26f2fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","failed_at":"alert","signature_id":"51d2fea77be7cecb840acdb29e21e26279922de433103aefa5ec401ed53ade1f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T01:22:06Z\n Ended: 2026-08-25T01:39:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2511104-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd9qo43pb87kgb0najuskd4463erq6p-s6z0k3y7v0o2v4a\", pod=\"router-794d4d4f76-jgglj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd9qo43pb87kgb0najuskd4463erq6p-s6z0k3y7v0o2v4a/router-794d4d4f76-jgglj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c63ab8f72d048872ab54b84458e40461465a3ddf24411a1bd4f28b50a54ae20b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","failed_at":"e2e","signature_id":"83e3baeadd89b998c4b81d1bc0c70d07aa9fec97f2694533c8ed2432ea266d46","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048360\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xjgfszrzk8fd\" to finish creating: GET https://rp.j2511104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5b520fd-de2d-4c04-817a-0cab0589d669\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5b520fd-de2d-4c04-817a-0cab0589d669\",\n \"name\": \"d5b520fd-de2d-4c04-817a-0cab0589d669\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T01:10:12.363594861Z\",\n \"endTime\": \"2026-08-25T01:29:20.471660293Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048360\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xjgfszrzk8fd\" to finish creating: GET https://rp.j2511104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5b520fd-de2d-4c04-817a-0cab0589d669\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5b520fd-de2d-4c04-817a-0cab0589d669\",\n \"name\": \"d5b520fd-de2d-4c04-817a-0cab0589d669\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T01:10:12.363594861Z\",\n \"endTime\": \"2026-08-25T01:29:20.471660293Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e280cd5ea909b84815488162050cbae280d92245364be0df93dfcf47adadbf45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","failed_at":"e2e","signature_id":"f1a6c7858d28eeb824a65797f9f24cbd2dea977f3e661a9e3c7f104a3dc601ee","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z4pnctblvmpf/cluster-candidate-4-22-bt5tfn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017de640\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bt5tfn: failed waiting for cluster=\"cluster-candidate-4-22-bt5tfn\" in resourcegroup=\"rg-candidate-4-22-z4pnctblvmpf\" to finish creating: GET https://rp.j2511104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8a2b3ac-1907-490d-8c60-3586709ab393\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8a2b3ac-1907-490d-8c60-3586709ab393\",\n \"name\": \"c8a2b3ac-1907-490d-8c60-3586709ab393\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T01:09:52.015224549Z\",\n \"endTime\": \"2026-08-25T01:29:00.623079296Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z4pnctblvmpf/cluster-candidate-4-22-bt5tfn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017de640\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bt5tfn: failed waiting for cluster=\"cluster-candidate-4-22-bt5tfn\" in resourcegroup=\"rg-candidate-4-22-z4pnctblvmpf\" to finish creating: GET https://rp.j2511104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8a2b3ac-1907-490d-8c60-3586709ab393\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8a2b3ac-1907-490d-8c60-3586709ab393\",\n \"name\": \"c8a2b3ac-1907-490d-8c60-3586709ab393\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T01:09:52.015224549Z\",\n \"endTime\": \"2026-08-25T01:29:00.623079296Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092041100716937216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-25T00:07:21Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c5c90a9ade150e8e70bf413a293cc1ab25d90ac1993f4974014669d31499f698","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092041100716937216","occurred_at":"2026-08-25T00:07:21Z","failed_at":"e2e","signature_id":"938c8e2babfc879ca08c00539016e48f5863191d883d5d8b5e82fc0bb437190d","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0008284f0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[01:06:30.008]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:27.0754516Z);\\nlet _endTime = datetime(2026-08-25T01:06:27.0754522Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:29.9876601Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212\\\",\\r\\n \\\"activityId\\\": \\\"59bb71ac-27cb-4cfe-91f1-0dc80dc01c78\\\",\\r\\n \\\"subActivityId\\\": \\\"a6ab6f7d-7448-47fd-b473-40d106fd5ddb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"df117fcb-5da2-4690-9e92-052d46a6d0e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212 ARID=59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.Http.CallContext/59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.ExecuteQuery/df117fcb-5da2-4690-9e92-052d46a6d0e1 \\u003e DN.FE.ExecuteQuery/a6ab6f7d-7448-47fd-b473-40d106fd5ddb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[01:06:30.057]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:27.0754516Z);\\nlet _endTime = datetime(2026-08-25T01:06:27.0754522Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:29.9876601Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212\\\",\\r\\n \\\"activityId\\\": \\\"59bb71ac-27cb-4cfe-91f1-0dc80dc01c78\\\",\\r\\n \\\"subActivityId\\\": \\\"a6ab6f7d-7448-47fd-b473-40d106fd5ddb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"df117fcb-5da2-4690-9e92-052d46a6d0e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212 ARID=59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.Http.CallContext/59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.ExecuteQuery/df117fcb-5da2-4690-9e92-052d46a6d0e1 \\u003e DN.FE.ExecuteQuery/a6ab6f7d-7448-47fd-b473-40d106fd5ddb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[01:06:33.062]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:30.1288605Z);\\nlet _endTime = datetime(2026-08-25T01:06:30.1288611Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:33.0424541Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6\\\",\\r\\n \\\"activityId\\\": \\\"d5604ed5-126a-4121-b965-3b7cbf0283f1\\\",\\r\\n \\\"subActivityId\\\": \\\"dd36c913-5740-44c2-bd2d-98a620b22e83\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"436ae237-897e-4d8a-81ee-c4b74c7b1d5b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6 ARID=d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.Http.CallContext/d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.ExecuteQuery/436ae237-897e-4d8a-81ee-c4b74c7b1d5b \\u003e DN.FE.ExecuteQuery/dd36c913-5740-44c2-bd2d-98a620b22e83)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[01:06:33.116]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:30.1288605Z);\\nlet _endTime = datetime(2026-08-25T01:06:30.1288611Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:33.0424541Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6\\\",\\r\\n \\\"activityId\\\": \\\"d5604ed5-126a-4121-b965-3b7cbf0283f1\\\",\\r\\n \\\"subActivityId\\\": \\\"dd36c913-5740-44c2-bd2d-98a620b22e83\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"436ae237-897e-4d8a-81ee-c4b74c7b1d5b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6 ARID=d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.Http.CallContext/d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.ExecuteQuery/436ae237-897e-4d8a-81ee-c4b74c7b1d5b \\u003e DN.FE.ExecuteQuery/dd36c913-5740-44c2-bd2d-98a620b22e83)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[01:06:39.536]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:36.3917754Z);\\nlet _endTime = datetime(2026-08-25T01:06:36.3917761Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:39.5170179Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c\\\",\\r\\n \\\"activityId\\\": \\\"52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2\\\",\\r\\n \\\"subActivityId\\\": \\\"3a59414c-fc4c-4995-b8b4-df986c791422\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"11c42a10-1df6-4d97-8ede-cf662bfe0997\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c ARID=52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.Http.CallContext/52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.ExecuteQuery/11c42a10-1df6-4d97-8ede-cf662bfe0997 \\u003e DN.FE.ExecuteQuery/3a59414c-fc4c-4995-b8b4-df986c791422)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[01:06:39.639]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:36.3917754Z);\\nlet _endTime = datetime(2026-08-25T01:06:36.3917761Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:39.5170179Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c\\\",\\r\\n \\\"activityId\\\": \\\"52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2\\\",\\r\\n \\\"subActivityId\\\": \\\"3a59414c-fc4c-4995-b8b4-df986c791422\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"11c42a10-1df6-4d97-8ede-cf662bfe0997\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c ARID=52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.Http.CallContext/52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.ExecuteQuery/11c42a10-1df6-4d97-8ede-cf662bfe0997 \\u003e DN.FE.ExecuteQuery/3a59414c-fc4c-4995-b8b4-df986c791422)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0008284f0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[01:06:30.008]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:27.0754516Z);\\nlet _endTime = datetime(2026-08-25T01:06:27.0754522Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:29.9876601Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212\\\",\\r\\n \\\"activityId\\\": \\\"59bb71ac-27cb-4cfe-91f1-0dc80dc01c78\\\",\\r\\n \\\"subActivityId\\\": \\\"a6ab6f7d-7448-47fd-b473-40d106fd5ddb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"df117fcb-5da2-4690-9e92-052d46a6d0e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212 ARID=59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.Http.CallContext/59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.ExecuteQuery/df117fcb-5da2-4690-9e92-052d46a6d0e1 \\u003e DN.FE.ExecuteQuery/a6ab6f7d-7448-47fd-b473-40d106fd5ddb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[01:06:30.057]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:27.0754516Z);\\nlet _endTime = datetime(2026-08-25T01:06:27.0754522Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:29.9876601Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212\\\",\\r\\n \\\"activityId\\\": \\\"59bb71ac-27cb-4cfe-91f1-0dc80dc01c78\\\",\\r\\n \\\"subActivityId\\\": \\\"a6ab6f7d-7448-47fd-b473-40d106fd5ddb\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"df117fcb-5da2-4690-9e92-052d46a6d0e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea9f9362-a343-4744-80b3-340fe3cac212 ARID=59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.Http.CallContext/59bb71ac-27cb-4cfe-91f1-0dc80dc01c78 \\u003e GW.ExecuteQuery/df117fcb-5da2-4690-9e92-052d46a6d0e1 \\u003e DN.FE.ExecuteQuery/a6ab6f7d-7448-47fd-b473-40d106fd5ddb)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[01:06:33.062]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:30.1288605Z);\\nlet _endTime = datetime(2026-08-25T01:06:30.1288611Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:33.0424541Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6\\\",\\r\\n \\\"activityId\\\": \\\"d5604ed5-126a-4121-b965-3b7cbf0283f1\\\",\\r\\n \\\"subActivityId\\\": \\\"dd36c913-5740-44c2-bd2d-98a620b22e83\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"436ae237-897e-4d8a-81ee-c4b74c7b1d5b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6 ARID=d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.Http.CallContext/d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.ExecuteQuery/436ae237-897e-4d8a-81ee-c4b74c7b1d5b \\u003e DN.FE.ExecuteQuery/dd36c913-5740-44c2-bd2d-98a620b22e83)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[01:06:33.116]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:30.1288605Z);\\nlet _endTime = datetime(2026-08-25T01:06:30.1288611Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:33.0424541Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6\\\",\\r\\n \\\"activityId\\\": \\\"d5604ed5-126a-4121-b965-3b7cbf0283f1\\\",\\r\\n \\\"subActivityId\\\": \\\"dd36c913-5740-44c2-bd2d-98a620b22e83\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"436ae237-897e-4d8a-81ee-c4b74c7b1d5b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;20e6c056-fef5-435a-98cd-ebba63cc75a6 ARID=d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.Http.CallContext/d5604ed5-126a-4121-b965-3b7cbf0283f1 \\u003e GW.ExecuteQuery/436ae237-897e-4d8a-81ee-c4b74c7b1d5b \\u003e DN.FE.ExecuteQuery/dd36c913-5740-44c2-bd2d-98a620b22e83)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[01:06:39.536]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:36.3917754Z);\\nlet _endTime = datetime(2026-08-25T01:06:36.3917761Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:39.5170179Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c\\\",\\r\\n \\\"activityId\\\": \\\"52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2\\\",\\r\\n \\\"subActivityId\\\": \\\"3a59414c-fc4c-4995-b8b4-df986c791422\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"11c42a10-1df6-4d97-8ede-cf662bfe0997\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c ARID=52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.Http.CallContext/52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.ExecuteQuery/11c42a10-1df6-4d97-8ede-cf662bfe0997 \\u003e DN.FE.ExecuteQuery/3a59414c-fc4c-4995-b8b4-df986c791422)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[01:06:39.639]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T01:06:36.3917754Z);\\nlet _endTime = datetime(2026-08-25T01:06:36.3917761Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xl7j9qn9wjl5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T01:06:39.5170179Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c\\\",\\r\\n \\\"activityId\\\": \\\"52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2\\\",\\r\\n \\\"subActivityId\\\": \\\"3a59414c-fc4c-4995-b8b4-df986c791422\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"11c42a10-1df6-4d97-8ede-cf662bfe0997\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26e617de-c0e4-4d37-b522-a570ccd2733c ARID=52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.Http.CallContext/52f0c2d0-c1d5-4598-b0a6-8ab0bda752b2 \\u003e GW.ExecuteQuery/11c42a10-1df6-4d97-8ede-cf662bfe0997 \\u003e DN.FE.ExecuteQuery/3a59414c-fc4c-4995-b8b4-df986c791422)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":1,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":0,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2092176584332021760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T09:05:43Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c1ee9b0e77e2c6fb01c2df0d80544c2b7826fbee226cc863ce48cbec5277eb31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2092176584332021760","occurred_at":"2026-08-25T09:05:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]},{"environment":"prod","summary":{"total_runs":9,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":5,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092318783510155264","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T18:30:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092275058721427456","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T15:37:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092270327135473664","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T15:18:13Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d49bef80fe38a92ab24f0c9978f25605850686531447b1b183a91bd64c643d7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092270327135473664","occurred_at":"2026-08-25T15:18:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092264779581231104","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T14:56:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092263269455630336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T14:50:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092192947897372672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T10:10:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"15e6bd55400a91d09a4546841ba23974a50ed7651a8342fc1f00c3677c9d5b07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092192947897372672","occurred_at":"2026-08-25T10:10:43Z","failed_at":"e2e","signature_id":"ad66320c5357dd06c4fac6f8b26348313ed54ba4b4268b4e00a204574fabedc1","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093afe0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-bm5z6ztxsxcf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\",\n \"name\": \"56042e31-0efc-4287-925a-c64efa21388d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:30:08.889821647Z\",\n \"endTime\": \"2026-08-25T10:49:13.717723647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093afe0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-bm5z6ztxsxcf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\",\n \"name\": \"56042e31-0efc-4287-925a-c64efa21388d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T10:30:08.889821647Z\",\n \"endTime\": \"2026-08-25T10:49:13.717723647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T07:06:40Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1c9f1b9de6126829475f5459cdee5602ca7612a61777b36d184bee15f4751444","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","occurred_at":"2026-08-25T07:06:40Z","failed_at":"e2e","signature_id":"16e9f20862be2ffa8d9254ea133442622a04c2230e897bc0c633345d949e6e0f","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d1700\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dnqrhz4h5n9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"name\": \"26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:34:58.109914662Z\",\n \"endTime\": \"2026-08-25T07:53:59.934466955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d1700\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dnqrhz4h5n9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"name\": \"26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:34:58.109914662Z\",\n \"endTime\": \"2026-08-25T07:53:59.934466955Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f8c5bb855fba9f1af1062b2e9b7dc3fc8cd18e3bdba61892b63fa7af886c6327","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","occurred_at":"2026-08-25T07:06:40Z","failed_at":"e2e","signature_id":"837b614da929eed13cefb96defae6f8e2c3ed1ab2105b3725cca3c17f8b63fa0","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a308e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ks6pxxnlhsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\",\n \"name\": \"fc3b6654-559a-489d-a192-908217a06b8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:33:44.141359446Z\",\n \"endTime\": \"2026-08-25T07:52:49.875150436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a308e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ks6pxxnlhsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\",\n \"name\": \"fc3b6654-559a-489d-a192-908217a06b8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T07:33:44.141359446Z\",\n \"endTime\": \"2026-08-25T07:52:49.875150436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T02:00:55Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0abbe5b518cd68bc3c4c369d878c96740039292fb9cfac4e318504273e484af8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","failed_at":"e2e","signature_id":"f812b5873fe0043e9314ca322d6d4251f24251a8ffe19fbea9df860ac64ea470","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zmwf7l76v4hv/cluster-candidate-5-0-p2dtvg should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000a42420\u003e: \n failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zmwf7l76v4hv/cluster-candidate-5-0-p2dtvg should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000a42420\u003e: \n failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded"}},{"row_id":"38537a27769f3a03f8133fb48d82c87fcdd4bc9d96fd64f0736e270aebbbffba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","failed_at":"e2e","signature_id":"47e6560075ab2811dc4afc63d80fcd866f7a8dbf54b0f240501f8704720bfe02","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-hc729b\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cc7e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4j8dsngwmfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-hc729b\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-hc729b\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cc7e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4j8dsngwmfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-hc729b\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.30; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"3ed866d86cc38758e16a12a9cd9e9139edb957b32e16b1035f77e2840cbe3518","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","failed_at":"e2e","signature_id":"328ab83e407af59a94692dc615a105004e707662f379710e72fd8156cdefd247","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000b1c2d0\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000b1c2d0\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5"}},{"row_id":"94d96f42b5f5859e1ec19ecf1651af181a78a66712a9222593b3011a75c02758","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","failed_at":"e2e","signature_id":"6b7decf97f628a3a86f9cc81aac7b2b793ce71dbec005f9c0bbc92ad65ed2200","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-2m6hjk\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00144a750\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6dbxgwtzxfsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2m6hjk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-2m6hjk\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00144a750\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6dbxgwtzxfsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2m6hjk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.35; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"cc774a95a4203136171e720d924821a9eddb4c1bd7374e7dc5583c31b957fb6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","failed_at":"e2e","signature_id":"5f57af3b362f707a2fd214f4c7abae7f575a8b66d80b93974a89ebabf4108803","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-qp64bq\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a2bec0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-dqpzmwzqshp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-qp64bq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-qp64bq\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a2bec0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-dqpzmwzqshp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-qp64bq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.10; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092058685026078720","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T01:17:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"stg","summary":{"total_runs":3,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":0,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2092108568042934272","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T04:35:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2092069681564225536","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-25T02:00:55Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5a598e707c392c432c4dec45e3ffb68f01a1a439e145087e93a8aa3decfeccdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2092069681564225536","occurred_at":"2026-08-25T02:00:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2092054307795898368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-25T00:59:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}