{"meta":{"date":"2026-09-27","timezone":"UTC","generated_at":"2026-10-14T16:30:32Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":30,"failed_runs":27,"runs_with_occurrences":27,"runs_with_failure_pattern_matches":26,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"c91d80458b129d5116e8c39c1b7bce8247abd84b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T22:20:12Z","started_at":"2026-09-27T22:20:12Z","completed_at":"2026-09-28T00:32:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T00:40:39.709562141Z","timing_metadata_checked_at":"2026-09-28T00:40:39.709562141Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T00:40:39.71276343Z","region_metadata_checked_at":"2026-09-28T00:40:39.71276343Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"12334a235c9bf97924f7fb7736a036cf0f30888f76a36090898c0564eceac798","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","failed_at":"e2e","signature_id":"e8a205541114f6cec3b425e2ad1adb4dc2be321629c4a93557e1f3b4a5460bb4","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000858480\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000858480\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5a32867416c248246850f053eb447f1cb93d8fba2bae3455ee41d0153f99c5c0","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": read tcp \u003cip\u003e-\u003e\u003cip\u003e: read: connection reset by peer","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer"}},{"row_id":"6aff2a71a1a7166e86900b1c080539a9dc285d6a666347be78d2ec4c4360c345","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","failed_at":"alert","signature_id":"419f780bdbaa43f4ef8f4ecb0e33396b8bfad0e38a30159c1ff2da81cde2f1a7","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T23:23:56Z\n  Ended:       2026-09-28T00:09:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j9480704-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j9480704-mgmt-1\nNamespace: ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"a0f2d8dd94ece23ec38328818cf1b291ed6cfaa949c7c27e9ca4fddefc6b23eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","failed_at":"alert","signature_id":"141dcb4a7bce83ddae2534a50b4037182ee14743f68296b8e0c644526062ca81","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T23:20:54Z\n  Ended:       2026-09-27T23:44:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j9480704-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j9480704-mgmt-1\nNamespace: ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"bcfd0fe11f90e3f3330355cf318c2120bb9a0f7290f8fc3e02fdb35a809ed72f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","failed_at":"e2e","signature_id":"01e7808196f8b35484085dcce2391cd7ff226b123e9d686fab532297c48cc3ba","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6d380\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-4tsj8j\" in resourcegroup=\"rg-np-noedge-4tsj8j-67tl4d7r7ztd\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"name\": \"36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:24:00.446121961Z\",\n      \"endTime\": \"2026-09-27T23:43:03.337836309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6d380\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-4tsj8j\" in resourcegroup=\"rg-np-noedge-4tsj8j-67tl4d7r7ztd\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"name\": \"36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:24:00.446121961Z\",\n      \"endTime\": \"2026-09-27T23:43:03.337836309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f9f14c402507557c27354fd441e02600eb708593103e9ca7d3457348694ee2f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","failed_at":"e2e","signature_id":"0edcc1c21b78bd45a5dcd2f988466a7ec1ce2a2761e3091d1504343efd9bb710","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060efe0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wz5nqx5brl4f\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"name\": \"3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:20:03.595087025Z\",\n      \"endTime\": \"2026-09-27T23:39:03.372293439Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-dq426: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-wz5nqx5brl4f--managed/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Network/networkInterfaces/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060efe0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wz5nqx5brl4f\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"name\": \"3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:20:03.595087025Z\",\n      \"endTime\": \"2026-09-27T23:39:03.372293439Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-dq426: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-wz5nqx5brl4f--managed/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Network/networkInterfaces/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","5a32867416c248246850f053eb447f1cb93d8fba2bae3455ee41d0153f99c5c0","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"8b6ab8cfca9e76601f0130042fe70cf1493f19d3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T21:19:10Z","started_at":"2026-09-27T21:19:10Z","completed_at":"2026-09-27T23:23:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T23:37:51.440231545Z","timing_metadata_checked_at":"2026-09-27T23:37:51.440231545Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T23:37:51.443227635Z","region_metadata_checked_at":"2026-09-27T23:37:51.443227635Z"},"failed_at":["alert","e2e"],"failed_test_count":11,"occurrences":[{"row_id":"1748cfafb132e5123df91a5f50387bf2e9bb329bd059f626e64bf6a5d87205dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"21db66633adda02d93dd92446231c93c6316598ee599dce2a1a985be29893e32","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013e0660\u003e: \n    failed to create HCP cluster np-dg-minor-z78ths: failed waiting for cluster=\"np-dg-minor-z78ths\" in resourcegroup=\"rg-np-dg-minor-z78ths-lxsmmmlnvmjv\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54291c20-e9b4-4070-9657-55e8e053432d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54291c20-e9b4-4070-9657-55e8e053432d\",\n      \"name\": \"54291c20-e9b4-4070-9657-55e8e053432d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T22:21:31.146099917Z\",\n      \"endTime\": \"2026-09-27T22:50:31.661815914Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013e0660\u003e: \n    failed to create HCP cluster np-dg-minor-z78ths: failed waiting for cluster=\"np-dg-minor-z78ths\" in resourcegroup=\"rg-np-dg-minor-z78ths-lxsmmmlnvmjv\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54291c20-e9b4-4070-9657-55e8e053432d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54291c20-e9b4-4070-9657-55e8e053432d\",\n      \"name\": \"54291c20-e9b4-4070-9657-55e8e053432d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T22:21:31.146099917Z\",\n      \"endTime\": \"2026-09-27T22:50:31.661815914Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"39f3512c6857b72b278564a335848e5505d1043f2f66b2cf3d548e6820aec3a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"alert","signature_id":"7c6c23658687695d208580c0620f85e276d1932e76e62b017338962064930246","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (6 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:06:28Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T22:06:29Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-27T22:07:30Z\n  Ended:       2026-09-27T22:48:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-27T22:07:30Z\n  Ended:       2026-09-27T22:48:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-27T22:09:27Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-27T22:10:30Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"4557efcfca6014c0729b2c3c6281a52d16a0c5f304e730e9d5580ea0a01258c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"3edaa5633377ee32cc909ec3f497506e56f88cc918442b0177acc1481df58f76","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-97fd8t2fpt8h/cluster-candidate-4-23-jvmr6j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106ed20\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-jvmr6j: failed waiting for cluster=\"cluster-candidate-4-23-jvmr6j\" in resourcegroup=\"rg-candidate-4-23-97fd8t2fpt8h\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1d51430-01dc-4d6c-9f0a-0b21f000ca8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1d51430-01dc-4d6c-9f0a-0b21f000ca8d\",\n      \"name\": \"d1d51430-01dc-4d6c-9f0a-0b21f000ca8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:36.712987842Z\",\n      \"endTime\": \"2026-09-27T22:26:36.653416533Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-97fd8t2fpt8h/cluster-candidate-4-23-jvmr6j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106ed20\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-jvmr6j: failed waiting for cluster=\"cluster-candidate-4-23-jvmr6j\" in resourcegroup=\"rg-candidate-4-23-97fd8t2fpt8h\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1d51430-01dc-4d6c-9f0a-0b21f000ca8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1d51430-01dc-4d6c-9f0a-0b21f000ca8d\",\n      \"name\": \"d1d51430-01dc-4d6c-9f0a-0b21f000ca8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:36.712987842Z\",\n      \"endTime\": \"2026-09-27T22:26:36.653416533Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6171246db0f996456e9be7e85b4a02dc3d14e2429ff1da5dea87ae3edfc11657","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"790de6b4868a370f0d3dba6636aa96ab453fbb116768e39345dfb5a27aee5c2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"056f5aac48261bf6a11c23178e7053c153d8a77c8cd07b47c6d8f44988ca9be3","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e260\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-hhxkjbgbq9fb\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"name\": \"3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.777648405Z\",\n      \"endTime\": \"2026-09-27T22:26:17.06664737Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e260\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-hhxkjbgbq9fb\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"name\": \"3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.777648405Z\",\n      \"endTime\": \"2026-09-27T22:26:17.06664737Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"92c312514797b8394b2aaf38d1ea89d67e28bec1e71a1a1042415b82074b737b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"0170db780c4df4adf7bb8a89ac92e967f775ba081325c4cac8d09effb77a4c27","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de2a20\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tcmg5wddbpw5\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"name\": \"75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.92972128Z\",\n      \"endTime\": \"2026-09-27T22:26:36.766928511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de2a20\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tcmg5wddbpw5\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"name\": \"75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.92972128Z\",\n      \"endTime\": \"2026-09-27T22:26:36.766928511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9666ad892ea3f0e0ddbad157fae4679cd80ded050a6b84d3b6de3bf0922a576d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"fe1e6109eb86fcb5bef47d66924fe8d64130a64ca2fed35ead60f414517a7f41","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bc7480\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fzbdj6z9fvsp\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ae4b814-345a-401a-ad49-b8ffd613eb42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ae4b814-345a-401a-ad49-b8ffd613eb42\",\n      \"name\": \"7ae4b814-345a-401a-ad49-b8ffd613eb42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:27.476971207Z\",\n      \"endTime\": \"2026-09-27T22:26:36.91910306Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bc7480\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fzbdj6z9fvsp\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ae4b814-345a-401a-ad49-b8ffd613eb42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ae4b814-345a-401a-ad49-b8ffd613eb42\",\n      \"name\": \"7ae4b814-345a-401a-ad49-b8ffd613eb42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:27.476971207Z\",\n      \"endTime\": \"2026-09-27T22:26:36.91910306Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"adb4e2a84bae78f8850cf51aca54a3214f93f4002350e3460f63a2a0a20d6122","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"alert","signature_id":"e00c24569b7fef8b6c9f87bbc44d7468ad1c94ae60292dc7180f5c34825d2871","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-27T22:19:32Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-27T22:19:32Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-27T22:20:28Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-27T22:20:28Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"d6427725060f76496f730bfdd070e95b17c7360921563fbca84b3e13388b232b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"14017b17f411363983778e1786bd28771dd63d57ce07d0aef5decd8c20e45c89","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a74820\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d7zqqvsfl6cw\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f90451ae-33e2-4eab-9a3e-26e39b48aef2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f90451ae-33e2-4eab-9a3e-26e39b48aef2\",\n      \"name\": \"f90451ae-33e2-4eab-9a3e-26e39b48aef2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.347881728Z\",\n      \"endTime\": \"2026-09-27T22:26:36.956451886Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a74820\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d7zqqvsfl6cw\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f90451ae-33e2-4eab-9a3e-26e39b48aef2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f90451ae-33e2-4eab-9a3e-26e39b48aef2\",\n      \"name\": \"f90451ae-33e2-4eab-9a3e-26e39b48aef2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.347881728Z\",\n      \"endTime\": \"2026-09-27T22:26:36.956451886Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e60d77f375ff96d0c551e95ef24b169851f5463a65b631dbb2c699f27d6d0ded","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"046677b799dbe94e2abdd61ad6c306ffc70f772329a705fd183eeddfe68167fe","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f767a0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tlzxr8jmz7kk\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"name\": \"4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.95420119Z\",\n      \"endTime\": \"2026-09-27T22:26:17.083195861Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f767a0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tlzxr8jmz7kk\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"name\": \"4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.95420119Z\",\n      \"endTime\": \"2026-09-27T22:26:17.083195861Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ef043fa2fbb8093031b5446e13d21c7361064e266a2c23c7944835a206e6b752","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","failed_at":"e2e","signature_id":"8fc8ffdd4987b337260088b62ea7c21d838b773930ca05d6c0bce06df0b64bba","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9f6a0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-tfnpjqsqltmz\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef11fb4d-c122-4f3d-8662-7a6be0a63439\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef11fb4d-c122-4f3d-8662-7a6be0a63439\",\n      \"name\": \"ef11fb4d-c122-4f3d-8662-7a6be0a63439\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:17.555091008Z\",\n      \"endTime\": \"2026-09-27T22:26:17.063491391Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9f6a0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-tfnpjqsqltmz\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef11fb4d-c122-4f3d-8662-7a6be0a63439\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef11fb4d-c122-4f3d-8662-7a6be0a63439\",\n      \"name\": \"ef11fb4d-c122-4f3d-8662-7a6be0a63439\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:17.555091008Z\",\n      \"endTime\": \"2026-09-27T22:26:17.063491391Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T21:02:53Z","started_at":"2026-09-27T21:02:53Z","completed_at":"2026-09-27T23:23:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T23:26:52.093160366Z","timing_metadata_checked_at":"2026-09-27T23:26:52.093160366Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T23:26:52.096808254Z","region_metadata_checked_at":"2026-09-27T23:26:52.096808254Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"1f9b40457981b7194debe8551abb5892007df65c5f37f4621f3f91be79f8212b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","occurred_at":"2026-09-27T21:02:53Z","failed_at":"alert","signature_id":"d6d997e13699f3708b8da2df30da87216f574a5d894c86f3487f4f6d6d9ec091","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:08:16Z\n  Ended:       2026-09-27T23:04:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2557696-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2557696-mgmt-2\nNamespace: ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"64e656723631d7227a1bc0ebe88ee85db7f43c9539bf153b34307559f99d24eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","occurred_at":"2026-09-27T21:02:53Z","failed_at":"alert","signature_id":"a4af37eae3845736c3bca8e48388195a8cf0fb39b9828c0799328f4f81338760","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:05:16Z\n  Ended:       2026-09-27T22:39:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2557696-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2557696-mgmt-2\nNamespace: ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7201/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291389238939648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7201,"pr_state":"closed","pr_sha":"1d70ed9d0eee138e4ae0bc5e6a6ea0e96da50848","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T19:25:37Z","started_at":"2026-09-27T19:25:37Z","completed_at":"2026-09-27T21:37:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T21:42:14.130964158Z","timing_metadata_checked_at":"2026-09-27T21:42:14.130964158Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T21:42:14.134103848Z","region_metadata_checked_at":"2026-09-27T21:42:14.134103848Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"27baffeba065d8d208314f91f8a485a64ca852db3921f613743c9c3529545c22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7201/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291389238939648","occurred_at":"2026-09-27T19:25:37Z","failed_at":"e2e","signature_id":"eea917c119fb496a73e78c0ccb01ffe97195fb9bbf05312d263c9d161c01f38e","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa8060\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kcdxbmxql2zw\" to finish creating: GET https://rp.j8939648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"name\": \"4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:35:22.462990652Z\",\n      \"endTime\": \"2026-09-27T20:54:31.807536747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa8060\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kcdxbmxql2zw\" to finish creating: GET https://rp.j8939648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"name\": \"4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:35:22.462990652Z\",\n      \"endTime\": \"2026-09-27T20:54:31.807536747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7200,"pr_state":"open","pr_sha":"7bdbf36ebd0ae544431133e8c0235ea4ee732c3c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T19:25:36Z","started_at":"2026-09-27T19:25:36Z","completed_at":"2026-09-27T21:42:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T21:52:23.316701785Z","timing_metadata_checked_at":"2026-09-27T21:52:23.316701785Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T21:52:23.319987976Z","region_metadata_checked_at":"2026-09-27T21:52:23.319987976Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"68c43e01ff8579c4a5f9f6ccc90bee63e07154cfcac4bbe352fbc988c0d7542c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","occurred_at":"2026-09-27T19:25:36Z","failed_at":"alert","signature_id":"80970b397031adb31bef780349e99ab505e3f2fd73c9134ca7e696f1e795203c","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T20:30:25Z\n  Ended:       2026-09-27T21:00:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T20:30:25Z\n  Ended:       2026-09-27T20:55:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"c275f6505ef766dd4c3eca13595b6b01a3397f79dd7e4a18f9451659262bed52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","occurred_at":"2026-09-27T19:25:36Z","failed_at":"alert","signature_id":"ba0d797453f4acd2d3294ab03019f050a2ea64a58bfda6df78da0690b02a2c50","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T20:33:23Z\n  Ended:       2026-09-27T21:25:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T20:33:23Z\n  Ended:       2026-09-27T21:20:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104290856126124032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7183,"pr_state":"closed","pr_sha":"af98c4cd096a77fe2288782a43b7f12151970e6a","final_merged_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T19:23:30Z","started_at":"2026-09-27T19:23:30Z","completed_at":"2026-09-27T21:37:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T21:42:13.957341511Z","timing_metadata_checked_at":"2026-09-27T21:42:13.957341511Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T21:42:13.965034687Z","region_metadata_checked_at":"2026-09-27T21:42:13.965034687Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0580650d4ed1158e76b2628ddae40fd353ae0ed82504b7feabee2bc7a815a401","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104290856126124032","occurred_at":"2026-09-27T19:23:30Z","failed_at":"e2e","signature_id":"a7885b2a5379fa2f9992b69c22b756f6bcde94e94685985bf119094349859833","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000f972a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-t7rmzspt8s2k\" to finish creating: GET https://rp.j6124032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"name\": \"0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:26:25.730010902Z\",\n      \"endTime\": \"2026-09-27T20:45:26.626992422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-t7rmzspt8s2k--managed/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-t7rmzspt8s2k--managed/providers/Microsoft.Network/networkInterfaces/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' seconds after additiona... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000f972a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-t7rmzspt8s2k\" to finish creating: GET https://rp.j6124032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"name\": \"0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:26:25.730010902Z\",\n      \"endTime\": \"2026-09-27T20:45:26.626992422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-t7rmzspt8s2k--managed/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-t7rmzspt8s2k--managed/providers/Microsoft.Network/networkInterfaces/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' seconds after additiona... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"84dce2d6dba2b1e1b30987861051d79e7081c5aa","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T18:30:51Z","started_at":"2026-09-27T18:30:51Z","completed_at":"2026-09-27T20:42:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T20:49:51.923423478Z","timing_metadata_checked_at":"2026-09-27T20:49:51.923423478Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T20:49:51.92613087Z","region_metadata_checked_at":"2026-09-27T20:49:51.92613087Z"},"failed_at":["alert"],"failed_test_count":4,"occurrences":[{"row_id":"02fc5c613ab429480cd8c4632071079f2b91d7c762131113213be452ccea229c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","failed_at":"alert","signature_id":"5afb798429b94f5c90e2fc1387a8558369362de6b6d0c8a51b724a0353effcc8","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:30:48Z\n  Ended:       2026-09-27T20:20:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j1590528-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3hrgvt17an29u42rn5461sqrrkuc36-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j1590528-mgmt-2\nNamespace: ocm-arohcpci01-2t3hrgvt17an29u42rn5461sqrrkuc36-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"8ad8cdc7f86205c346af5e8606a9934edfcb30f4a75f63a80c8e52874808e2f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","failed_at":"alert","signature_id":"191a6f3923b9adfd1af697a8104aad4e301e37d24962d749afe729bbd97e8f3a","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:28:50Z\n  Ended:       2026-09-27T20:12:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j1590528-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j1590528-mgmt-1\nNamespace: ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"b31cd30293771b0a5c2ad5fa8c9601772246a3a04259f56eb7cea6452a3b41bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","failed_at":"alert","signature_id":"426761ff9070cec28a5322afd36d3627552c3d22962de9e435913509b416aa8d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:27:47Z\n  Ended:       2026-09-27T19:55:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j1590528-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3hrgvt17an29u42rn5461sqrrkuc36-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1590528-mgmt-2\nNamespace: ocm-arohcpci01-2t3hrgvt17an29u42rn5461sqrrkuc36-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"ebccfa75cd8015c286bb034d5e52d72be899f82bca8135c462891966132f35b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","failed_at":"alert","signature_id":"893ce1f9fd178961fa552c9c4ed69700c8bd2c12d9b189640fba68210e5e2004","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:25:52Z\n  Ended:       2026-09-27T19:47:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j1590528-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1590528-mgmt-1\nNamespace: ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"8d7d77d27f5c4854822930057f9c8f7763b57b58","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T18:24:28Z","started_at":"2026-09-27T18:24:28Z","completed_at":"2026-09-27T20:41:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T20:49:51.79167741Z","timing_metadata_checked_at":"2026-09-27T20:49:51.79167741Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T20:49:51.795095799Z","region_metadata_checked_at":"2026-09-27T20:49:51.795095799Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"49e015b78928b6e67d1ed6b37f1402ec8a6709980f1420a8658ba6aaaf17962e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","failed_at":"alert","signature_id":"8d21629c67d82189d892875373e45a37e10d571da501c0c8f7adab114af4262c","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:29:26Z\n  Ended:       2026-09-27T20:13:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2756608-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3hq6389up01vpn0d0ugl7ajsd0kjv0-q8g1r5a9g2q9m7l\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2756608-mgmt-1\nNamespace: ocm-arohcpci01-2t3hq6389up01vpn0d0ugl7ajsd0kjv0-q8g1r5a9g2q9m7l","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"968f2dbf150a84c7306f1385759c5fe72bb622e3ffe9e6636ac60baf537ec436","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","failed_at":"alert","signature_id":"44a63252c6af03437dca2a2cd16633fffc635048800be390cd8ed0b750e906c9","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:26:26Z\n  Ended:       2026-09-27T19:48:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2756608-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3hq6389up01vpn0d0ugl7ajsd0kjv0-q8g1r5a9g2q9m7l\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2756608-mgmt-1\nNamespace: ocm-arohcpci01-2t3hq6389up01vpn0d0ugl7ajsd0kjv0-q8g1r5a9g2q9m7l","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"cfea2f718244e73d41a1a61cbfb635537e5ed47ca881841d275229bc87e4462d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104264194718175232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T17:37:33Z","started_at":"2026-09-27T17:37:33Z","completed_at":"2026-09-27T19:56:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T20:08:24.94608779Z","timing_metadata_checked_at":"2026-09-27T20:08:24.94608779Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T20:08:24.94915888Z","region_metadata_checked_at":"2026-09-27T20:08:24.94915888Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"5bd8b6fb541752254621750a1c55ba4410127895fb2d58357be6e44c36179e01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104264194718175232","occurred_at":"2026-09-27T17:37:33Z","failed_at":"alert","signature_id":"b1312384d9b2a53ea723368157d1cd9705e0e9f23816fc33749b5b53bb2fe8fb","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:49:46Z\n  Ended:       2026-09-27T18:56:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j8175232-svc\", component=\"fleet\", name=\"nodepoolcontroller\", region=\"westus3\", severity=\"warning\"\n  Description: Fleet controller workqueue nodepoolcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7183,"pr_state":"closed","pr_sha":"af98c4cd096a77fe2288782a43b7f12151970e6a","final_merged_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T17:03:43Z","started_at":"2026-09-27T17:03:43Z","completed_at":"2026-09-27T19:20:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T19:27:37.691379433Z","timing_metadata_checked_at":"2026-09-27T19:27:37.691379433Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T19:27:37.694422523Z","region_metadata_checked_at":"2026-09-27T19:27:37.694422523Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1f46062b9ace9232bc8331d4df4d899fbbff6d4429fc57a388c9cc50036f6bf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","failed_at":"alert","signature_id":"35bed69d107d0e48eeae3149c2b40959f5c177d8606a49c69653469eb84a80bc","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:07:35Z\n  Ended:       2026-09-27T18:36:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j6653824-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3gm692dov66okrgiqv3gkivt7lh79a-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6653824-mgmt-1\nNamespace: ocm-arohcpci01-2t3gm692dov66okrgiqv3gkivt7lh79a-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"3540c285a0e572f86d3873836d0dbe93df6fec2024ad867810c20e154e66e471","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","failed_at":"alert","signature_id":"5e5cd04319eed373eaaf914f96f5fe47f4a0a57f723cd12e6afda7269f6982ed","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:10:36Z\n  Ended:       2026-09-27T19:01:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j6653824-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3gm692dov66okrgiqv3gkivt7lh79a-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6653824-mgmt-1\nNamespace: ocm-arohcpci01-2t3gm692dov66okrgiqv3gkivt7lh79a-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"dd027a95c09d3d5b54ae4b6e3c022e13d765650bfec14c378465bd2283973906","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","failed_at":"e2e","signature_id":"05ebaa9488bd93eeb90e405c068beae1c93f0836a7ac6badc62d7e0b3eadd4c5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:133]: failed to get admin REST config for cluster \"cluster-candidate-4-21-cscwdf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001673c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-cscwdf\" in resourcegroup=\"rg-candidate-4-21-xbwd2trtds6r\" to finish getting creds: GET https://rp.j6653824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:133]: failed to get admin REST config for cluster \"cluster-candidate-4-21-cscwdf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001673c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-cscwdf\" in resourcegroup=\"rg-candidate-4-21-xbwd2trtds6r\" to finish getting creds: GET https://rp.j6653824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7199,"pr_state":"open","pr_sha":"d374a1a9ff79a5e9bad1859f0680a241e1b2d135","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T16:28:23Z","started_at":"2026-09-27T16:28:23Z","completed_at":"2026-09-27T18:48:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T18:55:52.146735026Z","timing_metadata_checked_at":"2026-09-27T19:06:37.881088253Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T18:55:52.150409514Z","region_metadata_checked_at":"2026-09-27T18:55:52.150409514Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"71525ae0604637eb32292b18df59c0b2fae469a014229d03973eb2c0651bbe22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","occurred_at":"2026-09-27T16:28:23Z","failed_at":"alert","signature_id":"bd2bb1667dcc46addf88c81f320c44761ca728d893e15805a3bd4ad0673a41d2","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:37:18Z\n  Ended:       2026-09-27T17:43:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1132032-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3g642vb3ovmhl4qiumvarv9qifaek4-c6e1u6h4n5k3a7j\", pod=\"router-5584fc994b-5x9hp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3g642vb3ovmhl4qiumvarv9qifaek4-c6e1u6h4n5k3a7j/router-5584fc994b-5x9hp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e4b8da2581c0c3fdc56026d0c12443e797d9244518c908f4e604978d5bf97f44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","occurred_at":"2026-09-27T16:28:23Z","failed_at":"alert","signature_id":"d18c749f3722a30d0857576cb4260959235b0821801cb56556539b905f76002f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:00:15Z\n  Ended:       2026-09-27T18:06:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1132032-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1132032-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7198,"pr_state":"closed","pr_sha":"83ddd740bad73c24d4d63a4b535c98cd472d7d12","final_merged_sha":"83ddd740bad73c24d4d63a4b535c98cd472d7d12","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T16:00:25Z","started_at":"2026-09-27T16:00:25Z","completed_at":"2026-09-27T18:09:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T18:13:22.152553464Z","timing_metadata_checked_at":"2026-09-27T18:13:22.152553464Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T18:13:22.156287252Z","region_metadata_checked_at":"2026-09-27T18:13:22.156287252Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"37262fe460d8347b1f0aaaec432cfd8386838f7a93c653b01f0671ad38baba43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","failed_at":"e2e","signature_id":"2d46f58172df00b26b08a908114c7df93ee1d42789e568fd90c7a7b65ba15046","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00089fe00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp.j5652992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"name\": \"6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T17:01:59.070822701Z\",\n      \"endTime\": \"2026-09-27T17:21:03.436757642Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00089fe00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp.j5652992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"name\": \"6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T17:01:59.070822701Z\",\n      \"endTime\": \"2026-09-27T17:21:03.436757642Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp"}},{"row_id":"6d5a259246a6b3f917fac1a985d8ebdbc72eaf1b471380003e173702a5332761","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","failed_at":"alert","signature_id":"f9c445cff424013d845cfdec607b448ab1198b54444d73458cb4dab5656a8299","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T16:57:42Z\n  Ended:       2026-09-27T17:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5652992-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5652992-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"704c3014e8a51cc083f302d787ac81d6369213318760965dc3ad64436a8b5e27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","failed_at":"alert","signature_id":"a3273e0d1794b7c8a67f767205c53f73d87ec1d167083395847a4aaed230b76f","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:14:22Z\n  Ended:       2026-09-27T17:26:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j5652992-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j5652992-mgmt-2\nNamespace: ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"82df5d8d8ddf76b115e6a9e672a20915b7791089b27ad8793f40d7c4e01ccfcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","failed_at":"alert","signature_id":"66132c0ef28f02375a925f830a63f89a139e8421a3d4bf3c8d7b497dcbd25498","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:21:23Z\n  Ended:       2026-09-27T17:51:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j5652992-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j5652992-mgmt-2\nNamespace: ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104208646610096128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"161d5208292dedf681b66d95ec683b93d5b9db92","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T13:56:50Z","started_at":"2026-09-27T13:56:50Z","completed_at":"2026-09-27T15:45:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T15:56:57.470938884Z","timing_metadata_checked_at":"2026-09-27T15:56:57.470938884Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T15:56:57.474143873Z","region_metadata_checked_at":"2026-09-27T15:56:57.474143873Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2157796b2f9db999cd103d4aee94a1276778c017881aa6cf2db09a2cb4fd4aad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104208646610096128","occurred_at":"2026-09-27T13:56:50Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201444725886976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7198,"pr_state":"closed","pr_sha":"b4090decf8dcdab2c0113599327eb0d5aa29138d","final_merged_sha":"83ddd740bad73c24d4d63a4b535c98cd472d7d12","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-27T13:28:13Z","started_at":"2026-09-27T13:28:13Z","completed_at":"2026-09-27T15:52:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T16:07:15.422658987Z","timing_metadata_checked_at":"2026-09-27T16:07:15.422658987Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T16:07:15.425974076Z","region_metadata_checked_at":"2026-09-27T16:07:15.425974076Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7197,"pr_state":"closed","pr_sha":"31a52ec9475c15f0f56d6c0690393f0afa49a12d","final_merged_sha":"31a52ec9475c15f0f56d6c0690393f0afa49a12d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T13:27:59Z","started_at":"2026-09-27T13:27:59Z","completed_at":"2026-09-27T15:46:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T15:56:57.599582716Z","timing_metadata_checked_at":"2026-09-27T15:56:57.599582716Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T15:56:57.602337306Z","region_metadata_checked_at":"2026-09-27T15:56:57.602337306Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"4e216489f9eab6a6442610ed4e68554d22d310a4613ce71b70afb5e0b2736a2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T14:40:20Z\n  Ended:       2026-09-27T14:57:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5467008-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"6d12d5b1efaa8269b5afbd4faff851eb393c360ebcf6800535f2182df385ec6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","failed_at":"alert","signature_id":"6a3860a8f043002633cbc86fed6af04ae8f3209a874cc185b949e92160d28645","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T14:38:39Z\n  Ended:       2026-09-27T15:26:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j5467008-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3di9vg5945pk406769u1t7vrpjin1b-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j5467008-mgmt-2\nNamespace: ocm-arohcpci01-2t3di9vg5945pk406769u1t7vrpjin1b-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"e525ac4d5150135054ee176296978fe85e56f2048ed6acc391acb8f8f921388f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","failed_at":"alert","signature_id":"bdc02d01d20a4113becec172cb87ce69502ed7be7f54534e201fa4c1be3941cb","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T14:35:35Z\n  Ended:       2026-09-27T15:01:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j5467008-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3di9vg5945pk406769u1t7vrpjin1b-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j5467008-mgmt-2\nNamespace: ocm-arohcpci01-2t3di9vg5945pk406769u1t7vrpjin1b-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104196521745453056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"0624ebea04735cf02f6ff6b91f00307730d915c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T13:08:39Z","started_at":"2026-09-27T13:08:39Z","completed_at":"2026-09-27T15:13:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T15:24:24.602742747Z","timing_metadata_checked_at":"2026-09-27T15:24:24.602742747Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T15:24:24.605439138Z","region_metadata_checked_at":"2026-09-27T15:24:24.605439138Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8c28eb1947614c7722229d22bfa96ec94126a83fb983fb306ddd7ee7fb4cfa09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104196521745453056","occurred_at":"2026-09-27T13:08:39Z","failed_at":"alert","signature_id":"325bf50ff4030e0a9e49b5267b69a16cbb66d95e03425b035183cf4895a04dc3","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (1 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T14:26:50Z\n  Ended:       2026-09-27T14:32:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5453056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3d20g44qhuupsokqk6foneks2ohpk7-agg-cluster\", pod=\"kube-apiserver-857c58cf4-hl9vw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3d20g44qhuupsokqk6foneks2ohpk7-agg-cluster/kube-apiserver-857c58cf4-hl9vw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104191454321577984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7196,"pr_state":"closed","pr_sha":"5936e37ed46aa9564d4efa2c3754415cc8f674de","final_merged_sha":"5936e37ed46aa9564d4efa2c3754415cc8f674de","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T12:48:31Z","started_at":"2026-09-27T12:48:31Z","completed_at":"2026-09-27T14:57:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T15:03:17.315704299Z","timing_metadata_checked_at":"2026-09-27T15:03:17.315704299Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T15:03:17.31825659Z","region_metadata_checked_at":"2026-09-27T15:03:17.31825659Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d95a0dcb2f4383673a8db6977a2a8c5ed833f8ccba7935c5d825f2324af0afcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104191454321577984","occurred_at":"2026-09-27T12:48:31Z","failed_at":"e2e","signature_id":"5926710b9fa5f78e3a02a2c8e840a39699891ffa25494a7619c4e511a8bddcd0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000676bd0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000676bd0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104191299589509120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"6c9b04c06b6c7c99f40cba07e0123ccd5def4f90","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-27T12:47:54Z","started_at":"2026-09-27T12:47:54Z","completed_at":"2026-09-27T14:54:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T15:03:17.153967165Z","timing_metadata_checked_at":"2026-09-27T15:03:17.153967165Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T15:03:17.157296153Z","region_metadata_checked_at":"2026-09-27T15:03:17.157296153Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104127032412606464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T08:32:31Z","started_at":"2026-09-27T08:32:31Z","completed_at":"2026-09-27T08:41:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T08:56:55.006184765Z","timing_metadata_checked_at":"2026-09-27T08:56:55.006184765Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-27T08:56:55.009546255Z","region_metadata_checked_at":"2026-09-27T09:59:06.333107383Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"92fb187100a9921bcb80d2950fbd10515b032f6c7a2de30e98457b3d0a3d292b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104127032412606464","occurred_at":"2026-09-27T08:32:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104124987630686208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"6cb9902d022bb3673764d0dcc05cb3d82bd22f51","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T08:24:24Z","started_at":"2026-09-27T08:24:24Z","completed_at":"2026-09-27T10:34:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T10:50:40.28628225Z","timing_metadata_checked_at":"2026-09-27T10:50:40.28628225Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T10:50:40.296641813Z","region_metadata_checked_at":"2026-09-27T10:50:40.296641813Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"df7ac7978d2a6f94ee93954691c76e7d5ca6ff9960c6881becf6c110b9a389e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104124987630686208","occurred_at":"2026-09-27T08:24:24Z","failed_at":"e2e","signature_id":"97240d8885a0bdfc86afa503aac43fee59507e22b5387aeb628e8d3e06faee04","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104123485520400384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7193,"pr_state":"closed","pr_sha":"c45a472db4e317b0509e3ff2987572e72e1f6031","final_merged_sha":"c45a472db4e317b0509e3ff2987572e72e1f6031","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T08:18:26Z","started_at":"2026-09-27T08:18:26Z","completed_at":"2026-09-27T10:16:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T10:29:54.120773745Z","timing_metadata_checked_at":"2026-09-27T10:29:54.120773745Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T10:29:54.124021936Z","region_metadata_checked_at":"2026-09-27T10:29:54.124021936Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e6403856e2c1289aac25ffde80772500879df0a35225ec6d3bb484d103fa2e87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104123485520400384","occurred_at":"2026-09-27T08:18:26Z","failed_at":"e2e","signature_id":"9b8aa1049a27cadfcf00f24cced762a2cb52851eacbb3b0c5ad6f8e5a3f78f90","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029105a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029105a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"183179ac11c500b20a5a115a52bcd2be4e647fd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T07:50:03Z","started_at":"2026-09-27T07:50:03Z","completed_at":"2026-09-27T09:57:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T10:09:25.960323658Z","timing_metadata_checked_at":"2026-09-27T10:09:25.960323658Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T10:09:25.963640948Z","region_metadata_checked_at":"2026-09-27T10:09:25.963640948Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2da38005dbe48f376de9995805b6ac70a9ea41e10442c19a798897300be8ff7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","failed_at":"e2e","signature_id":"0ef5b19fd1cdd2f1706442a99609b4839aa21fe049bf6ef3221a44afe9300590","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001692000\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-cgcqm9ph7z7k\" to finish creating: GET https://rp.j9166976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/960cce01-25b6-4e04-aeb2-169c0394ac37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/960cce01-25b6-4e04-aeb2-169c0394ac37\",\n      \"name\": \"960cce01-25b6-4e04-aeb2-169c0394ac37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T08:48:39.120988577Z\",\n      \"endTime\": \"2026-09-27T09:07:39.122832689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-m7pgk-gh8dm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cgcqm9ph7z7k--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-m7pgk-gh8dm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-m7pgk-gh8dm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cgcqm9ph7z7k--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-m7pgk-gh8dm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001692000\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-cgcqm9ph7z7k\" to finish creating: GET https://rp.j9166976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/960cce01-25b6-4e04-aeb2-169c0394ac37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/960cce01-25b6-4e04-aeb2-169c0394ac37\",\n      \"name\": \"960cce01-25b6-4e04-aeb2-169c0394ac37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T08:48:39.120988577Z\",\n      \"endTime\": \"2026-09-27T09:07:39.122832689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-m7pgk-gh8dm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cgcqm9ph7z7k--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-m7pgk-gh8dm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-m7pgk-gh8dm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cgcqm9ph7z7k--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-m7pgk-gh8dm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3c6ded9b577fd42b237d6e0d021c239bf9fcabb2d88b14e159d87dd70111a641","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","failed_at":"e2e","signature_id":"f11b81cb85742fe5ff7b9b961ba857992240995905349698dcbcbd8852080bc6","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cb7860\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-jmzm77\" in resourcegroup=\"rg-np-version-upgrade-jmzm77-gnwj7tdglbmr\" to finish creating: GET https://rp.j9166976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2bbfea9-d25c-4222-afe4-d2febbb37c31\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2bbfea9-d25c-4222-afe4-d2febbb37c31\",\n      \"name\": \"f2bbfea9-d25c-4222-afe4-d2febbb37c31\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T08:50:17.403277735Z\",\n      \"endTime\": \"2026-09-27T09:09:19.085896095Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-jmzm77-gnwj7tdglbmr-np-upgrade-jm-12b3e238/providers/Microsoft.Compute/virtualMachines/v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-jmzm77-gnwj7tdglbmr-np-upgrade-jm-12b3e238/providers/Microsoft.Compute/virtualMachines/v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cb7860\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-jmzm77\" in resourcegroup=\"rg-np-version-upgrade-jmzm77-gnwj7tdglbmr\" to finish creating: GET https://rp.j9166976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2bbfea9-d25c-4222-afe4-d2febbb37c31\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2bbfea9-d25c-4222-afe4-d2febbb37c31\",\n      \"name\": \"f2bbfea9-d25c-4222-afe4-d2febbb37c31\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T08:50:17.403277735Z\",\n      \"endTime\": \"2026-09-27T09:09:19.085896095Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-jmzm77-gnwj7tdglbmr-np-upgrade-jm-12b3e238/providers/Microsoft.Compute/virtualMachines/v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-jmzm77-gnwj7tdglbmr-np-upgrade-jm-12b3e238/providers/Microsoft.Compute/virtualMachines/v9s3g2z1w8y5m9a-npupgrade-4-20-vbtk2-dsvr7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bb5238278224fe18eb7990dbc1f714dbde9b95c50da0eb8bf2fc85fe21580556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","failed_at":"e2e","signature_id":"0263dc178beb748cec45d68667416149811aa14e2aff4db55448a33e6e63b5c7","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001793e90\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-99j2d9sd47k2\" to finish creating: GET https://rp.j9166976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f398cd92-c2d5-469e-8eca-cf2a30ef86ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f398cd92-c2d5-469e-8eca-cf2a30ef86ac\",\n      \"name\": \"f398cd92-c2d5-469e-8eca-cf2a30ef86ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T08:52:25.123568687Z\",\n      \"endTime\": \"2026-09-27T09:11:28.968750353Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vvmdb-sgtsq: Failed: availabilitysets failed to create or update. err: failed to get existing resource feature-aggregation-99j2d9sd47k2--managed/2t38glvvr2plvdov72ijndtosji9gdee_agg-cluster-agg-np-a-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-99j2d9sd47k2--managed/providers/Microsoft.Compute/availabilitySets/2t38glvvr2plvdov72ijndtosji9gdee_agg-cluster-agg-np-a-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001793e90\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-99j2d9sd47k2\" to finish creating: GET https://rp.j9166976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f398cd92-c2d5-469e-8eca-cf2a30ef86ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f398cd92-c2d5-469e-8eca-cf2a30ef86ac\",\n      \"name\": \"f398cd92-c2d5-469e-8eca-cf2a30ef86ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T08:52:25.123568687Z\",\n      \"endTime\": \"2026-09-27T09:11:28.968750353Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vvmdb-sgtsq: Failed: availabilitysets failed to create or update. err: failed to get existing resource feature-aggregation-99j2d9sd47k2--managed/2t38glvvr2plvdov72ijndtosji9gdee_agg-cluster-agg-np-a-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-99j2d9sd47k2--managed/providers/Microsoft.Compute/availabilitySets/2t38glvvr2plvdov72ijndtosji9gdee_agg-cluster-agg-np-a-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7174,"pr_state":"closed","pr_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","final_merged_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T07:22:15Z","started_at":"2026-09-27T07:22:15Z","completed_at":"2026-09-27T09:18:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T09:27:47.441933291Z","timing_metadata_checked_at":"2026-09-27T09:27:47.441933291Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T09:27:47.444831782Z","region_metadata_checked_at":"2026-09-27T09:27:47.444831782Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"732582f03946a58a4748097cca8c7b40699f729f6c47b31cb18814d1cc1a6c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","failed_at":"alert","signature_id":"9d4a34fd40b7a4768d4e4eabe4a4ec42749a225062d6842b119ed23e64fabb54","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T08:05:14Z\n  Ended:       2026-09-27T08:31:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0172672-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0172672-mgmt-1\nNamespace: ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"8d76eec088f873c710b21f1cfb1e4f5c8e9d6b8a47e9d23b757152b1f97b3b57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","failed_at":"alert","signature_id":"430c9f14f7bfbeb1599ee7c1c8f52ce30424b0cc5c567fd877a045c6cf4ab795","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T08:29:34Z\n  Ended:       2026-09-27T08:35:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0172672-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0172672-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"8df39b7455897da89f3c9ad3b5a9b6f6322a65839b285d82703e762d53f60c93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","failed_at":"alert","signature_id":"0ae2d2740aeab65c0927cc3b7b9751bc45171ae3ffe4aada6c1848ef04948ef5","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T08:08:19Z\n  Ended:       2026-09-27T08:56:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0172672-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0172672-mgmt-1\nNamespace: ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071498628075520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7188,"pr_state":"closed","pr_sha":"1a692710f89c9c1ffcd99d3b48a7b09acd258143","final_merged_sha":"1a692710f89c9c1ffcd99d3b48a7b09acd258143","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-27T04:51:51Z","started_at":"2026-09-27T04:51:51Z","completed_at":"2026-09-27T07:07:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T07:21:33.104557757Z","timing_metadata_checked_at":"2026-09-27T07:21:33.104557757Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T07:21:33.107383049Z","region_metadata_checked_at":"2026-09-27T07:21:33.107383049Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7187,"pr_state":"open","pr_sha":"bf7f46462c66b992eb0ed0e4ace7ca43fcd6d8a6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T04:51:29Z","started_at":"2026-09-27T04:51:29Z","completed_at":"2026-09-27T07:08:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T07:21:33.006511741Z","timing_metadata_checked_at":"2026-09-27T07:21:33.006511741Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T07:21:33.01043563Z","region_metadata_checked_at":"2026-09-27T07:21:33.01043563Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"58d6e33089ce95d4343a2b510f1b34d63afeeb6c5803b559bc392f57f02e56f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","occurred_at":"2026-09-27T04:51:29Z","failed_at":"e2e","signature_id":"f0dd05bcedb68800765b2105c3e810ca2e4be249bca3b12d32e008a16b7acfdb","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002177290\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002177290\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"bb327eafe95505993b5bbcd4d0b5cef8200d3191582bbdb853617a0de5b40e37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","occurred_at":"2026-09-27T04:51:29Z","failed_at":"alert","signature_id":"c24afa2c3a89b1db96f561ea699ffc16b6ecf7c1b6c7babc01f986b2e2f32c4a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T06:19:21Z\n  Ended:       2026-09-27T06:25:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9712256-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9712256-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7174,"pr_state":"closed","pr_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","final_merged_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T04:50:58Z","started_at":"2026-09-27T04:50:58Z","completed_at":"2026-09-27T07:08:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T07:21:32.8480051Z","timing_metadata_checked_at":"2026-09-27T07:21:32.8480051Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T07:21:32.852331287Z","region_metadata_checked_at":"2026-09-27T07:21:32.852331287Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"3a4d237fb09b73ffbeaf61ecd6ded73c2f8de985c7b94fcd8516106075ed0124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","failed_at":"alert","signature_id":"6493d354b73970d4e764ed97c72ddea6bbaa29dc9a60565e51aa0b12f093243d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T05:56:50Z\n  Ended:       2026-09-27T06:14:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2686976-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2686976-mgmt-2\nNamespace: ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"572b42fb25250d6adca54cc0481cf6b7a7e57bad5c568d6b952d2cc0dbb2dc32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","failed_at":"alert","signature_id":"1a89df4c7c180c8de0615a9daa2555665cfe23d16df3d8e06611837f74f549ed","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T06:17:28Z\n  Ended:       2026-09-27T06:26:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2686976-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2686976-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"74545b1f050cd0e106a69550a9be8bb885c26267d29e7232b8d636f1188de54b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","failed_at":"alert","signature_id":"92318fcfac24b6233ea2adc5b233527abdcb8533537932bdc38c9c6b9dd86ffc","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T05:59:52Z\n  Ended:       2026-09-27T06:43:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2686976-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2686976-mgmt-2\nNamespace: ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"88b048d351cc4a5d496db393da1eb56356175482dbc3ac24fc4c19f9b239431f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","failed_at":"e2e","signature_id":"673f5d1216f0f7b060fd38c36ae28fe2306c5398d3ffd8fc404991258602f0b7","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000906ee0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5n7q2p9clx8c\" to finish creating: GET https://rp.j2686976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fec5206a-4b8c-473b-a0b3-7fd919567d02\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fec5206a-4b8c-473b-a0b3-7fd919567d02\",\n      \"name\": \"fec5206a-4b8c-473b-a0b3-7fd919567d02\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:55:45.853039376Z\",\n      \"endTime\": \"2026-09-27T06:14:46.209421265Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o4g6s6i3z3o6t4d-np-1-gj46m-mnj9b: Failed: availabilitysets failed to create or update. err: failed to get existing resource autoscaling-cluster-5n7q2p9clx8c--managed/2t35su4i1486ad8biqf1t6jj8eevkeio_o4g6s6i3z3o6t4d-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5n7q2p9clx8c--managed/providers/Microsoft.Compute/availabilitySets/2t35su4i1486ad8biqf1t6jj8eevkeio_o4g6s6i3z3o6t4d-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000906ee0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5n7q2p9clx8c\" to finish creating: GET https://rp.j2686976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fec5206a-4b8c-473b-a0b3-7fd919567d02\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fec5206a-4b8c-473b-a0b3-7fd919567d02\",\n      \"name\": \"fec5206a-4b8c-473b-a0b3-7fd919567d02\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:55:45.853039376Z\",\n      \"endTime\": \"2026-09-27T06:14:46.209421265Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o4g6s6i3z3o6t4d-np-1-gj46m-mnj9b: Failed: availabilitysets failed to create or update. err: failed to get existing resource autoscaling-cluster-5n7q2p9clx8c--managed/2t35su4i1486ad8biqf1t6jj8eevkeio_o4g6s6i3z3o6t4d-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5n7q2p9clx8c--managed/providers/Microsoft.Compute/availabilitySets/2t35su4i1486ad8biqf1t6jj8eevkeio_o4g6s6i3z3o6t4d-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a240e086a4cc64619a539f015ed07a3ccef6dc4aafc95c23f32040c854f5771f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","failed_at":"alert","signature_id":"0bafa741a68390f4b6bf980e8d38b5adb37e713e43141fb95567a5f642ddcfdd","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T05:55:51Z\n  Ended:       2026-09-27T06:20:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2686976-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2686976-mgmt-2\nNamespace: ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"edeb3e5ff7bf6467411977bc7766b9c94f74f98c752a7ec06118422917bc7e11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","failed_at":"alert","signature_id":"c1605d1431fa38cffcc889bc3b160bbc5879105dd589db8bc2b28a285e29f8b6","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T05:58:53Z\n  Ended:       2026-09-27T06:45:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2686976-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2686976-mgmt-2\nNamespace: ocm-arohcpci01-2t35ss5fcakhougflv038gatblofrv6a-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T04:49:39Z","started_at":"2026-09-27T04:49:39Z","completed_at":"2026-09-27T07:02:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T07:11:14.000268816Z","timing_metadata_checked_at":"2026-09-27T07:11:14.000268816Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T07:11:14.003726405Z","region_metadata_checked_at":"2026-09-27T07:11:14.003726405Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"63dc60321a32ff64daa2afcfbfd19ba542450484c9bdca77cd3d065ad4c0e8ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","failed_at":"alert","signature_id":"33e8f05c23ee905d13012921f05b90827ce9b99ceb5852cf28e896b69118e778","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T05:53:03Z\n  Ended:       2026-09-27T06:39:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j7860224-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t35rsiharub78pcihjajhlq044jn0tm-p6u8t9u3z0f3e0k\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j7860224-mgmt-1\nNamespace: ocm-arohcpci01-2t35rsiharub78pcihjajhlq044jn0tm-p6u8t9u3z0f3e0k","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"d1e0138a60e7979d4fafbb50cd540a681f0cf7bf3cf1fb79ae16cd94af5309bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","failed_at":"alert","signature_id":"c79fa012a0b071c6064bdd63844fb3a758f0f6a2bc109b1ebc88b8a973287f91","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T05:50:05Z\n  Ended:       2026-09-27T06:14:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j7860224-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t35rsiharub78pcihjajhlq044jn0tm-p6u8t9u3z0f3e0k\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j7860224-mgmt-1\nNamespace: ocm-arohcpci01-2t35rsiharub78pcihjajhlq044jn0tm-p6u8t9u3z0f3e0k","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"f02caf0960e2c504d924ec10d9351e0803daed763bf267e8b49bc2ccbf9dcbac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","failed_at":"e2e","signature_id":"ec4257f3d7d834907b3d94066f4a938edbf0824cf3b15c1781d4153ac05d1938","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a2980\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-5fskdvpkxfwb\" to finish creating: GET https://rp.j7860224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"name\": \"2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:48:02.556455592Z\",\n      \"endTime\": \"2026-09-27T06:07:09.276038359Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2: Failed: interfaces failed to create or update. err: failed to get existing resource admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/providers/Microsoft.Network/networkInterfaces/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a2980\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-5fskdvpkxfwb\" to finish creating: GET https://rp.j7860224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"name\": \"2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:48:02.556455592Z\",\n      \"endTime\": \"2026-09-27T06:07:09.276038359Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2: Failed: interfaces failed to create or update. err: failed to get existing resource admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/providers/Microsoft.Network/networkInterfaces/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070538413477888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7194,"pr_state":"open","pr_sha":"58636c0eb083ac2fd3924da89cbbe44e1edd0d23","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T04:48:02Z","started_at":"2026-09-27T04:48:02Z","completed_at":"2026-09-27T07:09:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T07:21:33.268545883Z","timing_metadata_checked_at":"2026-09-27T07:21:33.268545883Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T07:21:33.271445874Z","region_metadata_checked_at":"2026-09-27T07:21:33.271445874Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9b4022c99e71356744258bc523d3fae283fa7dad1a03c9aeb4005930e3c7fd3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070538413477888","occurred_at":"2026-09-27T04:48:02Z","failed_at":"alert","signature_id":"84013e20db593dd86936f009da2bff278404d255c4473789a9a242cf0c678207","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T06:16:41Z\n  Ended:       2026-09-27T06:22:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3477888-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3477888-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104062724328656896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"291b4dbd8cd0080905e0de9512d7581d8d4be844","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T04:16:59Z","started_at":"2026-09-27T04:16:59Z","completed_at":"2026-09-27T05:08:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T05:16:31.105143493Z","timing_metadata_checked_at":"2026-09-27T05:16:31.105143493Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T05:16:31.107856684Z","region_metadata_checked_at":"2026-09-27T05:16:31.107856684Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b3438e7129848ee5e881cacbdfffe900f6d7505782dadb8584bbb2be7a7f8e1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104062724328656896","occurred_at":"2026-09-27T04:16:59Z","failed_at":"provision","signature_id":"24248bfb9172326e331f0bc70fbaea3432445ca771da2ecd7e9aa55cd2f3ee9a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kusto-grant-ingest (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-27T04:55:22.142Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 description=\"Step kusto-grant-ingest\\n  Kind: ARM\\n  Template: templates/kusto-grant-ingest.bicep\\n  Parameters: configurations/kusto-mgmt-ingest.tmpl.bicepparam\"\ntime=2026-09-27T04:55:22.662Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2\ntime=2026-09-27T04:55:23.741Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 deployment=2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8656896-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80\ntime=2026-09-27T04:59:54.839Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8656896-mgmt-2/providers/Microsoft.Resources/deployments/2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80/operationStatuses/08584111219623969246\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/grantKusto-hcp-ja2yqt7pclopi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-kusto-us/providers/Microsoft.Kusto/clusters/hcp-dev-us-2/databases/HostedControlPlaneLogs/principalAssignments/grant-b6fd691c-fba0-50f9-bb53-13af017771ec\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Internal Server Error\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56fd2feac0bf2fd733880f128445ced8b465f2cded0a3ea3fc3aa25b8699b45c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Failed; provider Microsoft.Kusto","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["56fd2feac0bf2fd733880f128445ced8b465f2cded0a3ea3fc3aa25b8699b45c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104002487378776064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"6c9b04c06b6c7c99f40cba07e0123ccd5def4f90","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-27T00:17:38Z","started_at":"2026-09-27T00:17:38Z","completed_at":"2026-09-27T02:26:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T02:38:29.326688902Z","timing_metadata_checked_at":"2026-09-27T02:38:29.326688902Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T02:38:29.339322959Z","region_metadata_checked_at":"2026-09-27T02:38:29.339322959Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"909211bed4bd05b1c57701ffb54b85b5f6e2418bd0cc4f438af788acaef6832a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104002487378776064","occurred_at":"2026-09-27T00:17:38Z","failed_at":"alert","signature_id":"194da11ceda26cf004e70e30811c51ee51ab4977a0682ac50eca40b8c54129c3","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T01:21:42Z\n  Ended:       2026-09-27T01:35:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8776064-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T01:22:43Z\n  Ended:       2026-09-27T01:37:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8776064-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104259297394298880","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-27T17:18:06Z","started_at":"2026-09-27T17:18:06Z","completed_at":"2026-09-27T19:36:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T19:47:43.694338295Z","timing_metadata_checked_at":"2026-09-27T19:47:43.694338295Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T19:47:43.698396082Z","region_metadata_checked_at":"2026-09-27T19:47:43.698396082Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104153618864148480","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-27T10:18:10Z","started_at":"2026-09-27T10:18:10Z","completed_at":"2026-09-27T12:37:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T12:46:41.431084148Z","timing_metadata_checked_at":"2026-09-27T12:46:41.431084148Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T12:46:41.437645124Z","region_metadata_checked_at":"2026-09-27T12:46:41.437645124Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104104984143466496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T07:04:55Z","started_at":"2026-09-27T07:04:55Z","completed_at":"2026-09-27T09:14:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T09:27:47.460536733Z","timing_metadata_checked_at":"2026-09-27T09:27:47.460536733Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T09:27:47.463889922Z","region_metadata_checked_at":"2026-09-27T09:27:47.463889922Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1235614c14b7f59a177a11ff891c907c19b663002da91cefe10dd5019a58412e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104104984143466496","occurred_at":"2026-09-27T07:04:55Z","failed_at":"e2e","signature_id":"931488020c0f76286d54766d64d2f2259ec91252cd34366384a920f857d3befd","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00116fc20\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kj25nlz5xd2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"name\": \"f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T07:29:57.086711017Z\",\n      \"endTime\": \"2026-09-27T07:48:57.003548378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00116fc20\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kj25nlz5xd2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"name\": \"f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T07:29:57.086711017Z\",\n      \"endTime\": \"2026-09-27T07:48:57.003548378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T23:00:25Z","started_at":"2026-09-27T23:00:25Z","completed_at":"2026-09-28T01:59:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:05:23.895745168Z","timing_metadata_checked_at":"2026-09-28T02:05:23.895745168Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:05:23.898899057Z","region_metadata_checked_at":"2026-09-28T02:05:23.898899057Z"},"failed_at":["e2e"],"failed_test_count":40,"occurrences":[{"row_id":"10fc3045c4d7f36bab61769a7a45b82f24038716b872ef9646c1bf5e76b5c766","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"461696ab84dd3f6f5d834c5c1b6fcf988c3286f80243cf0e4e0b3877884cad1f","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfe8e0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pz8l8grcdj98\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55967661-6f28-4965-8bcc-cb1836294e43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55967661-6f28-4965-8bcc-cb1836294e43\",\n      \"name\": \"55967661-6f28-4965-8bcc-cb1836294e43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:37.37348914Z\",\n      \"endTime\": \"2026-09-27T23:31:45.970076258Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfe8e0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pz8l8grcdj98\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55967661-6f28-4965-8bcc-cb1836294e43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55967661-6f28-4965-8bcc-cb1836294e43\",\n      \"name\": \"55967661-6f28-4965-8bcc-cb1836294e43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:37.37348914Z\",\n      \"endTime\": \"2026-09-27T23:31:45.970076258Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"15135d3cb5fe58ff4fc7ad99950a4bc516fedcba323cd4e7805e5bd47e67ccce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"7026298ac0b13e5919540ccf258871ea869dd1bad766be2b0773591c9914e4b0","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002ba0720\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4pp72c\" resourcegroup=\"rg-np-dg-minor-4pp72c-t57mnh5thxv5\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4pp72c-t57mnh5thxv5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4pp72c/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002ba0720\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4pp72c\" resourcegroup=\"rg-np-dg-minor-4pp72c-t57mnh5thxv5\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4pp72c-t57mnh5thxv5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4pp72c/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1554718d4505615582b6a3eaf9f82e66b456eba07f6317a4899072f68ba17db8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2fc187761095416d0b74514b2e03a845bc6e9789fe8167aea732a57b59785d7b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012944a0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jgmdt9882d2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/468a566e-d147-4e1c-96c1-38bddafc0b6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/468a566e-d147-4e1c-96c1-38bddafc0b6f\",\n      \"name\": \"468a566e-d147-4e1c-96c1-38bddafc0b6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:17.0613341Z\",\n      \"endTime\": \"2026-09-27T23:31:25.968220027Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012944a0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jgmdt9882d2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/468a566e-d147-4e1c-96c1-38bddafc0b6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/468a566e-d147-4e1c-96c1-38bddafc0b6f\",\n      \"name\": \"468a566e-d147-4e1c-96c1-38bddafc0b6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:17.0613341Z\",\n      \"endTime\": \"2026-09-27T23:31:25.968220027Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1ae700c39805d2f33ad85f7e575032dd1c6b83f21c2f4f536e68f4d76835603a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"cbfeba8437d8617dc3f95e1a802b22d1f2cc72e90e32e533cc4a5aefd6974feb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a81100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9zjchk\" in resourcegroup=\"rg-np-version-upgrade-9zjchk-r4lv9m9d98td\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9zjchk-r4lv9m9d98td/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-9zjchk/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a81100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9zjchk\" in resourcegroup=\"rg-np-version-upgrade-9zjchk-r4lv9m9d98td\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9zjchk-r4lv9m9d98td/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-9zjchk/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1f96fe19ad055510cb2dad0e5cf4288df46049ed7263cd0c1cf65c8750271c4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"1bb95cbda5b42a403cf5850291c1bcd05d17dd2fca6b7e2a40ac7641b5dcb5b8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014b70f8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014b70f8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"31dc1f6aa2193b905dbaef14ec3671ebb87bf02951aa6ed299fe89c8b6e4136c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2e693ea3a6a3d55ed57f39150d24a058191d22a6fcfcea4aebe8f0d0bcf9f539","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0015174b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0015174b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"42579ed323c86a8583ab7f68e6fddd167fd2be24121c118868ae605752ddd70d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"b982657b8b51d839bd1a6e80552bb477e8899a650c3d88c5037ab958c4b34126","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b50d60\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tgc7nslklnzl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1235efb8-3cf2-4da7-8e61-25e5c2bc0a76\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1235efb8-3cf2-4da7-8e61-25e5c2bc0a76\",\n      \"name\": \"1235efb8-3cf2-4da7-8e61-25e5c2bc0a76\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:36.662922226Z\",\n      \"endTime\": \"2026-09-27T23:31:45.970102186Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b50d60\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tgc7nslklnzl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1235efb8-3cf2-4da7-8e61-25e5c2bc0a76\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1235efb8-3cf2-4da7-8e61-25e5c2bc0a76\",\n      \"name\": \"1235efb8-3cf2-4da7-8e61-25e5c2bc0a76\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:36.662922226Z\",\n      \"endTime\": \"2026-09-27T23:31:45.970102186Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"44f9e092d8f446e87cbebb82020bf864156ca24b11d0d3ddfbd0b5f99cf35b6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"650a5236bfcf5b14da373d69e1a2c8e2d365bee0c47a5d780593d397e4c57145","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eac560\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-b68r69hdkgbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c06c0022-17cf-4599-9905-c52d0766bab4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c06c0022-17cf-4599-9905-c52d0766bab4\",\n      \"name\": \"c06c0022-17cf-4599-9905-c52d0766bab4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:11.057895557Z\",\n      \"endTime\": \"2026-09-27T23:34:15.977577559Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eac560\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-b68r69hdkgbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c06c0022-17cf-4599-9905-c52d0766bab4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c06c0022-17cf-4599-9905-c52d0766bab4\",\n      \"name\": \"c06c0022-17cf-4599-9905-c52d0766bab4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:11.057895557Z\",\n      \"endTime\": \"2026-09-27T23:34:15.977577559Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"477e1765e9eaf62b5c54a345adcb03f8da5a37b79d8690195934cbefad494a65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2c4a64bd1771fc9b48285b77345bcc0b66100e0b2b369f902d0ab7f1dc30b0fb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tssqn7\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d6f80\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-tssqn7: failed waiting for cluster=\"cp-ystream-upgrade-5-0-tssqn7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-s5ql4kptw4ct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"name\": \"8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:40.978774448Z\",\n      \"endTime\": \"2026-09-28T00:26:43.612304644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tssqn7\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d6f80\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-tssqn7: failed waiting for cluster=\"cp-ystream-upgrade-5-0-tssqn7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-s5ql4kptw4ct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"name\": \"8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:40.978774448Z\",\n      \"endTime\": \"2026-09-28T00:26:43.612304644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1450b085cba9e38b0c42178fa8dd0c598b3bf57ce6c74d9b3dfd6c592b42cfc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c1e080c79a1254530370e13ef10e848d0cc1142df4c86d2b4b3613179d17789","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2b8afe759c56d8a7a8c15ff5b208b75d9c57c67b66b8b03a83b7df762b3540ad","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c7e0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7gqfj78cg774\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1741b83-b940-4f9f-958f-83b4e796a4f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1741b83-b940-4f9f-958f-83b4e796a4f6\",\n      \"name\": \"d1741b83-b940-4f9f-958f-83b4e796a4f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:18.821083127Z\",\n      \"endTime\": \"2026-09-28T00:28:23.573800876Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c7e0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7gqfj78cg774\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1741b83-b940-4f9f-958f-83b4e796a4f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1741b83-b940-4f9f-958f-83b4e796a4f6\",\n      \"name\": \"d1741b83-b940-4f9f-958f-83b4e796a4f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:18.821083127Z\",\n      \"endTime\": \"2026-09-28T00:28:23.573800876Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4e8d98341772ed82e12cc6a69ce5bf91242660b06b7fd4d37e5d89e0095489ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"33bb4455c170f5b024d933dc8204c4aa021b105023501e716ca4e138345d98ac","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010585e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7h98mb66pjnm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5af845a7-7502-4bd6-897b-19467c8a0ada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5af845a7-7502-4bd6-897b-19467c8a0ada\",\n      \"name\": \"5af845a7-7502-4bd6-897b-19467c8a0ada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:52.078427171Z\",\n      \"endTime\": \"2026-09-27T23:33:55.976360396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010585e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7h98mb66pjnm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5af845a7-7502-4bd6-897b-19467c8a0ada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5af845a7-7502-4bd6-897b-19467c8a0ada\",\n      \"name\": \"5af845a7-7502-4bd6-897b-19467c8a0ada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:52.078427171Z\",\n      \"endTime\": \"2026-09-27T23:33:55.976360396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4ed6501876937974083be62800b8147c29a5397ff394194feb28d2370b7a8937","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"a9e1b9127914b42c7adad7624b11d42e888367c10a014dfe93150ff03893a4cb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-btngfc\" for cluster \"cluster-nightly-4-20-btngfc\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4c1e0\u003e: \n    failed to create NodePool np-btngfc: failed starting nodepool creation \"np-btngfc\" for cluster \"cluster-nightly-4-20-btngfc\" in resourcegroup=\"rg-nightly-4-20-dlkdt65gwjpj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-dlkdt65gwjpj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-btngfc/nodePools/np-btngfc\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-btngfc\" for cluster \"cluster-nightly-4-20-btngfc\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4c1e0\u003e: \n    failed to create NodePool np-btngfc: failed starting nodepool creation \"np-btngfc\" for cluster \"cluster-nightly-4-20-btngfc\" in resourcegroup=\"rg-nightly-4-20-dlkdt65gwjpj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-dlkdt65gwjpj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-btngfc/nodePools/np-btngfc\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"66cfbb0e786db6357cb58f59a96fbd562388a7308283006da8dbda664471acd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"03d18401392997c3274ba2bf78e810311803b87bc7892a064ed600daf1058f86","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0026392a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pndxf7r8t9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"name\": \"a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:13:40.174684426Z\",\n      \"endTime\": \"2026-09-28T00:42:43.628005146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0026392a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pndxf7r8t9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"name\": \"a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:13:40.174684426Z\",\n      \"endTime\": \"2026-09-28T00:42:43.628005146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"678cb62a1603115fa90c919be8cb5ff986e51305925cda73e83c153038fe1679","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"1ab642fad09693398e82235c858d45a1797714580df61abc6d1b19d3918b71a4","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992660\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-nwqdmg89c6p9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"name\": \"7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:36.690582806Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971437914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992660\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-nwqdmg89c6p9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"name\": \"7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:36.690582806Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971437914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"67e57f4f4776021479d7e5cc779cb15ad6cad08b96e19e9c95bae4fe0b6d4f55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"60bff82014486820406a93416912dd354a1c71981703f4a298836390037dba3f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001068760\u003e: \n    failed to create HCP cluster ea-listjxmbqs: failed waiting for cluster=\"ea-listjxmbqs\" in resourcegroup=\"ea-list-hdczj2g88mkl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/91bd5de8-aa42-4e7a-b546-48535f13c304\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/91bd5de8-aa42-4e7a-b546-48535f13c304\",\n      \"name\": \"91bd5de8-aa42-4e7a-b546-48535f13c304\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:57.04687797Z\",\n      \"endTime\": \"2026-09-28T00:28:03.571739169Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001068760\u003e: \n    failed to create HCP cluster ea-listjxmbqs: failed waiting for cluster=\"ea-listjxmbqs\" in resourcegroup=\"ea-list-hdczj2g88mkl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/91bd5de8-aa42-4e7a-b546-48535f13c304\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/91bd5de8-aa42-4e7a-b546-48535f13c304\",\n      \"name\": \"91bd5de8-aa42-4e7a-b546-48535f13c304\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:57.04687797Z\",\n      \"endTime\": \"2026-09-28T00:28:03.571739169Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a62f8233ee555ea92f67ad7c4c824d9886cc9706629c0ceb1ee4046302c3d94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"6ae5ca364d6100ceef84da7c983fcde591de89d92c71a98ee689f822e2415b43","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0014c0f60\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/75179e80-a77b-471b-9ea6-7fe700fba9de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/75179e80-a77b-471b-9ea6-7fe700fba9de\",\n      \"name\": \"75179e80-a77b-471b-9ea6-7fe700fba9de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:07.551463967Z\",\n      \"endTime\": \"2026-09-27T23:32:15.972919815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0014c0f60\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/75179e80-a77b-471b-9ea6-7fe700fba9de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/75179e80-a77b-471b-9ea6-7fe700fba9de\",\n      \"name\": \"75179e80-a77b-471b-9ea6-7fe700fba9de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:07.551463967Z\",\n      \"endTime\": \"2026-09-27T23:32:15.972919815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"71619f9e3ea211c057a77430eb48c40ce3c7a77e56cdb08a48606965f94e7127","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"1ca48fafa9f0edecfb3787a894dc476ef91d87d36f79d5aa9e05a3fa12000657","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001014fe0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-c85djlsc4gqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a11dc30e-24bb-4e65-a64f-d5c4a10261d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a11dc30e-24bb-4e65-a64f-d5c4a10261d0\",\n      \"name\": \"a11dc30e-24bb-4e65-a64f-d5c4a10261d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:30.190180423Z\",\n      \"endTime\": \"2026-09-28T00:28:33.57420533Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001014fe0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-c85djlsc4gqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a11dc30e-24bb-4e65-a64f-d5c4a10261d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a11dc30e-24bb-4e65-a64f-d5c4a10261d0\",\n      \"name\": \"a11dc30e-24bb-4e65-a64f-d5c4a10261d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:30.190180423Z\",\n      \"endTime\": \"2026-09-28T00:28:33.57420533Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"775b991be6c2f92101aa9423759475ea380270615a97ad3dcd6134ddfa497cd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2bee03d5fc4053026371b7f338b599922d58ca513c59a44839dfe20827418271","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-q2v9b92t5wtw/cluster-nightly-4-21-255l2v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100e740\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-255l2v: failed waiting for cluster=\"cluster-nightly-4-21-255l2v\" in resourcegroup=\"rg-nightly-4-21-q2v9b92t5wtw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04aa15fa-a787-49ca-95e4-fb62f01ffcb1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04aa15fa-a787-49ca-95e4-fb62f01ffcb1\",\n      \"name\": \"04aa15fa-a787-49ca-95e4-fb62f01ffcb1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:08.296306358Z\",\n      \"endTime\": \"2026-09-28T00:26:13.568890404Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-q2v9b92t5wtw/cluster-nightly-4-21-255l2v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100e740\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-255l2v: failed waiting for cluster=\"cluster-nightly-4-21-255l2v\" in resourcegroup=\"rg-nightly-4-21-q2v9b92t5wtw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04aa15fa-a787-49ca-95e4-fb62f01ffcb1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04aa15fa-a787-49ca-95e4-fb62f01ffcb1\",\n      \"name\": \"04aa15fa-a787-49ca-95e4-fb62f01ffcb1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:08.296306358Z\",\n      \"endTime\": \"2026-09-28T00:26:13.568890404Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a55975d449e838e590919a9ab41cdc18c369354b64f6e128184fa25a0dfd71d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"6f750e9463768b9049eeacb07c97390e2b8b9fff31ce5357d95b8e0b486676c7","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc6900\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ppfrxnhlq6zz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a75d97c-2b17-4c11-9379-14857f7a504f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a75d97c-2b17-4c11-9379-14857f7a504f\",\n      \"name\": \"7a75d97c-2b17-4c11-9379-14857f7a504f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:37.656261821Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971502523Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc6900\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ppfrxnhlq6zz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a75d97c-2b17-4c11-9379-14857f7a504f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a75d97c-2b17-4c11-9379-14857f7a504f\",\n      \"name\": \"7a75d97c-2b17-4c11-9379-14857f7a504f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:37.656261821Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971502523Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fbbdb105eeb4b486697ddcebf5d146aa9529d83ff7fc53181c261d32d20aea9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"ddc46e7b679376d943aca50730e23da423333de42aa64aeea50b2982c7c8b320","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00083a9e0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7pdd9b2qtsjh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55bd848b-764d-4e14-887a-5c8e865b8b02\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55bd848b-764d-4e14-887a-5c8e865b8b02\",\n      \"name\": \"55bd848b-764d-4e14-887a-5c8e865b8b02\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:38.240422282Z\",\n      \"endTime\": \"2026-09-27T23:31:45.97016158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00083a9e0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7pdd9b2qtsjh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55bd848b-764d-4e14-887a-5c8e865b8b02\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55bd848b-764d-4e14-887a-5c8e865b8b02\",\n      \"name\": \"55bd848b-764d-4e14-887a-5c8e865b8b02\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:38.240422282Z\",\n      \"endTime\": \"2026-09-27T23:31:45.97016158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b8203f2d42ac561242fa039e7be8dee2d75ccec56f30b3a0dc4313e1b76675b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"f5e23bc64ae36da787717026bf1c86a8e8b33beccc9c6849ac470b476a3e7ddc","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-fbjrkv\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000adaa00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-fbjrkv: failed waiting for cluster=\"cp-ystream-upgrade-4-23-fbjrkv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xs76x2g6dgbl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"name\": \"2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:39.891778199Z\",\n      \"endTime\": \"2026-09-28T00:26:43.608164604Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-fbjrkv\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000adaa00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-fbjrkv: failed waiting for cluster=\"cp-ystream-upgrade-4-23-fbjrkv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xs76x2g6dgbl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"name\": \"2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:39.891778199Z\",\n      \"endTime\": \"2026-09-28T00:26:43.608164604Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a684e8587462e4ae472fc6b519009bd80aa09ce89fd7c5f1b3bac35c6a66cf86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"0ebe8a5d8973b947fbdd076fb748fedd8a7ffc4012adba74283db57c6063bad9","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866340\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wg62gnrkcq82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"name\": \"38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:28.308542215Z\",\n      \"endTime\": \"2026-09-27T23:31:35.969190413Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866340\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wg62gnrkcq82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"name\": \"38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:28.308542215Z\",\n      \"endTime\": \"2026-09-27T23:31:35.969190413Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a7dced8e04dddb0bb4054d8d6ef280c39b88c10084fd08663df0bc198b570e7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"65288eca0965be6dbcfe206e0aca14c0c0ca1eac6d17fd4f41aea772375b869d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-4jr4wg\" at install version 4.21.0-0.nightly-multi-2026-08-16-064216\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9e860\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-4jr4wg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-4jr4wg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-8vztd747qdlm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40ab65c1-e843-47e1-be96-78d0820ab888\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40ab65c1-e843-47e1-be96-78d0820ab888\",\n      \"name\": \"40ab65c1-e843-47e1-be96-78d0820ab888\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.974586451Z\",\n      \"endTime\": \"2026-09-28T00:26:23.56725042Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-4jr4wg\" at install version 4.21.0-0.nightly-multi-2026-08-16-064216\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9e860\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-4jr4wg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-4jr4wg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-8vztd747qdlm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40ab65c1-e843-47e1-be96-78d0820ab888\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40ab65c1-e843-47e1-be96-78d0820ab888\",\n      \"name\": \"40ab65c1-e843-47e1-be96-78d0820ab888\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.974586451Z\",\n      \"endTime\": \"2026-09-28T00:26:23.56725042Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b348fe41330e801599bfefceff22884aed941d2f0a26b3be1e15c2239e5c888a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"5853de0191a9e066d96eb6447d7e06566e036f05860c98d4f1c042f9f060842e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c38ae0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mqcj6sgbzwzl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4562cc54-0f79-4e54-9f42-579f7446258f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4562cc54-0f79-4e54-9f42-579f7446258f\",\n      \"name\": \"4562cc54-0f79-4e54-9f42-579f7446258f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:27.392724975Z\",\n      \"endTime\": \"2026-09-27T23:31:35.969501842Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c38ae0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mqcj6sgbzwzl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4562cc54-0f79-4e54-9f42-579f7446258f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4562cc54-0f79-4e54-9f42-579f7446258f\",\n      \"name\": \"4562cc54-0f79-4e54-9f42-579f7446258f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:27.392724975Z\",\n      \"endTime\": \"2026-09-27T23:31:35.969501842Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b96631f68a46c2d83f58cb50957c3b50aa6c6d5fc339f718cb11e92818db95dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"64d463d42f7127948f5a0d3f204437431ff48e5601f36b43f3827e98c0d20698","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gtjvj95k94qz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/340b72ab-640f-4ead-8482-0990a3a001a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/340b72ab-640f-4ead-8482-0990a3a001a4\",\n      \"name\": \"340b72ab-640f-4ead-8482-0990a3a001a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:11.801487325Z\",\n      \"endTime\": \"2026-09-27T23:33:15.975699789Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gtjvj95k94qz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/340b72ab-640f-4ead-8482-0990a3a001a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/340b72ab-640f-4ead-8482-0990a3a001a4\",\n      \"name\": \"340b72ab-640f-4ead-8482-0990a3a001a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:11.801487325Z\",\n      \"endTime\": \"2026-09-27T23:33:15.975699789Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba704d6fba18f8a0e00460d29ff8c8c5b56e1b39f07d5ff679eefe7b0b1fa476","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"f9162987953ca17cf343f465cb28a946cb34440c9d858e66ef18592f3f2b461b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56bc0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hffn6kx6ns6p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d3c213fe-3996-4b11-8d9a-d57af133e8ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d3c213fe-3996-4b11-8d9a-d57af133e8ac\",\n      \"name\": \"d3c213fe-3996-4b11-8d9a-d57af133e8ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:47.483810554Z\",\n      \"endTime\": \"2026-09-28T00:27:53.571341129Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56bc0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hffn6kx6ns6p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d3c213fe-3996-4b11-8d9a-d57af133e8ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d3c213fe-3996-4b11-8d9a-d57af133e8ac\",\n      \"name\": \"d3c213fe-3996-4b11-8d9a-d57af133e8ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:47.483810554Z\",\n      \"endTime\": \"2026-09-28T00:27:53.571341129Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c22bad02081b7be7cc4cd291c5dc7e001871d5f682b14a2b64f4d4b41caa7888","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"fb749055d31dcc0dc83a0c66aae251a4681c7b41862f59b30547d03392614d99","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc300\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-pj47jh\" in resourcegroup=\"rg-np-skip-minor-pj47jh-jmcx69spqbgl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-pj47jh-jmcx69spqbgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-pj47jh/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc300\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-pj47jh\" in resourcegroup=\"rg-np-skip-minor-pj47jh-jmcx69spqbgl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-pj47jh-jmcx69spqbgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-pj47jh/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c38842cc1c90b48d0140cfbcb6da116ca022a42053433dbd554de679519248b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"5f5245c97fdbd17ad8ac15e3e2445694b4f8d2a8f15128ca7457652d20c85a4b","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001014500\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-gdl957wjhnmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61551070-5bea-4c95-8cd2-94223dd885de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61551070-5bea-4c95-8cd2-94223dd885de\",\n      \"name\": \"61551070-5bea-4c95-8cd2-94223dd885de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:37.300565328Z\",\n      \"endTime\": \"2026-09-27T23:31:45.970879971Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001014500\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-gdl957wjhnmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61551070-5bea-4c95-8cd2-94223dd885de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61551070-5bea-4c95-8cd2-94223dd885de\",\n      \"name\": \"61551070-5bea-4c95-8cd2-94223dd885de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:37.300565328Z\",\n      \"endTime\": \"2026-09-27T23:31:45.970879971Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c4263f1cf2c82bbd87a9968fd27984b407b19498598eefac6a6472c7c682af8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"b94e5cf2e7c1fb55ced86979ddc7046ca3a21ac3308be4a35685174df4fcff94","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f96920\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-z49h7x9dg72j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efbc8d98-3e3c-4be8-a742-fceb1317ce9e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efbc8d98-3e3c-4be8-a742-fceb1317ce9e\",\n      \"name\": \"efbc8d98-3e3c-4be8-a742-fceb1317ce9e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:11.709436376Z\",\n      \"endTime\": \"2026-09-27T23:34:15.976995944Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f96920\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-z49h7x9dg72j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efbc8d98-3e3c-4be8-a742-fceb1317ce9e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efbc8d98-3e3c-4be8-a742-fceb1317ce9e\",\n      \"name\": \"efbc8d98-3e3c-4be8-a742-fceb1317ce9e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:11.709436376Z\",\n      \"endTime\": \"2026-09-27T23:34:15.976995944Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c508ef865fb7d919bdea2d4d0959a6e990f88ad004e384fc888ee17c7032823e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"d60bab05ea469af575a8a75ee3c255b1b84c6cf63eeb43e333195545bae9492e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c53960\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-klr6qxl7tgzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/721a00dd-f703-40c1-b1bd-8a22d7beaa2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/721a00dd-f703-40c1-b1bd-8a22d7beaa2b\",\n      \"name\": \"721a00dd-f703-40c1-b1bd-8a22d7beaa2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:30.932322501Z\",\n      \"endTime\": \"2026-09-27T23:32:35.974576236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c53960\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-klr6qxl7tgzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/721a00dd-f703-40c1-b1bd-8a22d7beaa2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/721a00dd-f703-40c1-b1bd-8a22d7beaa2b\",\n      \"name\": \"721a00dd-f703-40c1-b1bd-8a22d7beaa2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:30.932322501Z\",\n      \"endTime\": \"2026-09-27T23:32:35.974576236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c7b7bd6e8b1a4dac42de0c712942f3c98a9cfa951da83c9831713e93030e7f8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"c1f77dd01af7df31876c36d16401e499d9a695be8e5fd9fbfafb98309347710a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-j6wfl7rfmftf/cluster-nightly-4-22-nm6bdd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b18ea0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-nm6bdd: failed waiting for cluster=\"cluster-nightly-4-22-nm6bdd\" in resourcegroup=\"rg-nightly-4-22-j6wfl7rfmftf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"name\": \"30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.981848903Z\",\n      \"endTime\": \"2026-09-28T00:26:23.602322446Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-j6wfl7rfmftf/cluster-nightly-4-22-nm6bdd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b18ea0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-nm6bdd: failed waiting for cluster=\"cluster-nightly-4-22-nm6bdd\" in resourcegroup=\"rg-nightly-4-22-j6wfl7rfmftf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"name\": \"30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.981848903Z\",\n      \"endTime\": \"2026-09-28T00:26:23.602322446Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9a26912f46c9dd2f2e99d49a193342e8cce83c70b6ac4442b33b7adcf05653f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"22046caa0147e84e4bf00172320247011b3cb7426164354cc5c2ff6648d4106e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-mvb2mp2h8jkn/cluster-nightly-5-1-bjw648 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001076960\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-bjw648: failed waiting for cluster=\"cluster-nightly-5-1-bjw648\" in resourcegroup=\"rg-nightly-5-1-mvb2mp2h8jkn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a050b64c-6ea3-4b3d-bd9c-59727e5486cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a050b64c-6ea3-4b3d-bd9c-59727e5486cc\",\n      \"name\": \"a050b64c-6ea3-4b3d-bd9c-59727e5486cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.688455869Z\",\n      \"endTime\": \"2026-09-28T00:26:23.566602132Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-mvb2mp2h8jkn/cluster-nightly-5-1-bjw648 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001076960\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-bjw648: failed waiting for cluster=\"cluster-nightly-5-1-bjw648\" in resourcegroup=\"rg-nightly-5-1-mvb2mp2h8jkn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a050b64c-6ea3-4b3d-bd9c-59727e5486cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a050b64c-6ea3-4b3d-bd9c-59727e5486cc\",\n      \"name\": \"a050b64c-6ea3-4b3d-bd9c-59727e5486cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.688455869Z\",\n      \"endTime\": \"2026-09-28T00:26:23.566602132Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cf1ba9e0c16c7e2cda2dd9fcbeb4a4a851a367336938937134ac20d6214361c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"ad9b6981d10e4fb3c5ba70acecc09e110752fb556f8b94cbbd533a7e4f924b6f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000feaa80\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-ss6k2z4f5mr9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dffefc9-9875-40d1-ba0f-58f0a11d1a91\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dffefc9-9875-40d1-ba0f-58f0a11d1a91\",\n      \"name\": \"2dffefc9-9875-40d1-ba0f-58f0a11d1a91\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:12.795655613Z\",\n      \"endTime\": \"2026-09-27T23:33:15.9764747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000feaa80\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-ss6k2z4f5mr9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dffefc9-9875-40d1-ba0f-58f0a11d1a91\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2dffefc9-9875-40d1-ba0f-58f0a11d1a91\",\n      \"name\": \"2dffefc9-9875-40d1-ba0f-58f0a11d1a91\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:12.795655613Z\",\n      \"endTime\": \"2026-09-27T23:33:15.9764747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e0b23889b384dab500cc6abca3b4a0ddcd8fc22878117e77cafec37935e42153","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"e76c31373fba429a4e0c89223d50ccb6f8fcffab754ca1d91bcc970647ccd398","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"63a069ecf46f44ce769fd7002561795b40f053bb56e3e21dd6e759403fa2f18c","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8f440\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-k5f54rh29tw9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2695f42e-c81b-4e4b-95ae-936410e8590b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2695f42e-c81b-4e4b-95ae-936410e8590b\",\n      \"name\": \"2695f42e-c81b-4e4b-95ae-936410e8590b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:19:47.758187128Z\",\n      \"endTime\": \"2026-09-28T00:48:53.658149566Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8f440\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-k5f54rh29tw9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2695f42e-c81b-4e4b-95ae-936410e8590b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2695f42e-c81b-4e4b-95ae-936410e8590b\",\n      \"name\": \"2695f42e-c81b-4e4b-95ae-936410e8590b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:19:47.758187128Z\",\n      \"endTime\": \"2026-09-28T00:48:53.658149566Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eae3baa1f49027d1c29a735da30cdbf984a0437bb2acd410ed0f9afcb1bd4e8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2ddf0eb41f7f4835c32a0934a2b734e3cff4bfce66b8ba7fa3315db46958dfc7","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088760\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-zxmzb298997v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/92501662-6df7-42a3-aa43-bcd5971c9ba9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/92501662-6df7-42a3-aa43-bcd5971c9ba9\",\n      \"name\": \"92501662-6df7-42a3-aa43-bcd5971c9ba9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:56:24.733972836Z\",\n      \"endTime\": \"2026-09-28T00:25:33.565873088Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088760\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-zxmzb298997v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/92501662-6df7-42a3-aa43-bcd5971c9ba9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/92501662-6df7-42a3-aa43-bcd5971c9ba9\",\n      \"name\": \"92501662-6df7-42a3-aa43-bcd5971c9ba9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:56:24.733972836Z\",\n      \"endTime\": \"2026-09-28T00:25:33.565873088Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ecd090cdb68cac90ef1cc947ecde44cac01284b58349725a74cf35952602ee65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"514d013204cdac81f773e4ab4576f5c8edc422d2a1cdb915bd7a4b0bfecf2094","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxwdj8sw and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-fct27gcqstbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007a08f0\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxwdj8sw and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-fct27gcqstbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007a08f0\u003e: \n    storage account log verification failed\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"eef258063096458c7bc21a42edca38a520a9c50a721dda2170f3da18fe0f3643","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"c41ccfc8ae07559bda7c04e5a227db6555888492c62d50c7b225a44f527e69aa","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a90f20\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-nsqwrgcngwlr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ffcb232-74e7-4ea1-baf9-e6e3b272dd4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ffcb232-74e7-4ea1-baf9-e6e3b272dd4d\",\n      \"name\": \"4ffcb232-74e7-4ea1-baf9-e6e3b272dd4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:38.058655228Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971610274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a90f20\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-nsqwrgcngwlr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ffcb232-74e7-4ea1-baf9-e6e3b272dd4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ffcb232-74e7-4ea1-baf9-e6e3b272dd4d\",\n      \"name\": \"4ffcb232-74e7-4ea1-baf9-e6e3b272dd4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:38.058655228Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971610274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f792a1aa4172e494c3ee6a027a0dc3eb24b8b45d59812b722186782aa078d610","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2e82467c69bc3d4c0bfce22bf493cfb693f196befe788f76a6d3957522e3f5c2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001037020\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nbnflj48n7st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2fc333a9-a61b-4943-8047-7c9a4ad8b0d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2fc333a9-a61b-4943-8047-7c9a4ad8b0d9\",\n      \"name\": \"2fc333a9-a61b-4943-8047-7c9a4ad8b0d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:56:24.530831437Z\",\n      \"endTime\": \"2026-09-28T00:25:33.566328828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001037020\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nbnflj48n7st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2fc333a9-a61b-4943-8047-7c9a4ad8b0d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2fc333a9-a61b-4943-8047-7c9a4ad8b0d9\",\n      \"name\": \"2fc333a9-a61b-4943-8047-7c9a4ad8b0d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:56:24.530831437Z\",\n      \"endTime\": \"2026-09-28T00:25:33.566328828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f9e3f61fc1b9ab8af718bf212e08a002eda6d4bb0c458276f81a79826644f35e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":40,"failure_pattern_ids":["1450b085cba9e38b0c42178fa8dd0c598b3bf57ce6c74d9b3dfd6c592b42cfc9","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62"],"matched_occurrences":40,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104028667012190208","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T02:00:24Z","started_at":"2026-09-27T02:00:24Z","completed_at":"2026-09-27T04:56:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T05:05:51.28424777Z","timing_metadata_checked_at":"2026-09-27T05:05:51.28424777Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T05:05:51.290182152Z","region_metadata_checked_at":"2026-09-27T05:05:51.290182152Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"59963a45d6e2d7d977105ca185a386385f5a54a6819491261a08ce77d498b7ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104028667012190208","occurred_at":"2026-09-27T02:00:24Z","failed_at":"e2e","signature_id":"ab3a35949bad5bc410ebcfaade95c153a7cb24c117efaed240cca8cf81980e41","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13b60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-6k9kth8pvmf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"name\": \"eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:13:34.441508798Z\",\n      \"endTime\": \"2026-09-27T02:32:39.228262555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13b60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-6k9kth8pvmf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"name\": \"eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:13:34.441508798Z\",\n      \"endTime\": \"2026-09-27T02:32:39.228262555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d03c857df87915a1f313a0002be9251061d1538583bd5ab8de4fe51e677c5370","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104028667012190208","occurred_at":"2026-09-27T02:00:24Z","failed_at":"e2e","signature_id":"e2c8651eb00e230b2d9dc94cadb3e408c87a5a41e20a9cad8673027e24eb6ca8","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxt4mfpn and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-vg2849dzv4lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0004c67b0\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxt4mfpn and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-vg2849dzv4lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0004c67b0\u003e: \n    storage account log verification failed\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T23:00:25Z","started_at":"2026-09-27T23:00:25Z","completed_at":"2026-09-28T01:07:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T01:12:22.201130452Z","timing_metadata_checked_at":"2026-09-28T01:12:22.201130452Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T01:12:22.204223542Z","region_metadata_checked_at":"2026-09-28T01:12:22.204223542Z"},"failed_at":["e2e"],"failed_test_count":46,"occurrences":[{"row_id":"016a8102ccda1128c08f3907762e09b1b249653f4f22ae2dc481724058d721d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"882cd20f2a88ec92ee19a96ebdb0e7450566486d7063b764f2d65682dcbe6990","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106a880\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-6snh6pj58f7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83fb43bb-0f59-41e0-9a1b-8448655aa26b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83fb43bb-0f59-41e0-9a1b-8448655aa26b\",\n      \"name\": \"83fb43bb-0f59-41e0-9a1b-8448655aa26b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:39.021853494Z\",\n      \"endTime\": \"2026-09-27T23:31:39.487220639Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106a880\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-6snh6pj58f7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83fb43bb-0f59-41e0-9a1b-8448655aa26b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83fb43bb-0f59-41e0-9a1b-8448655aa26b\",\n      \"name\": \"83fb43bb-0f59-41e0-9a1b-8448655aa26b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:39.021853494Z\",\n      \"endTime\": \"2026-09-27T23:31:39.487220639Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"02ce57d697a81375d87f8082527114aed6d695ad4c71ef9763b4944390333b88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0e52789a9254af02bf84a6572690557e6e470276175202a83ecb44c6d1cf3d1","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"0c4da79ca7ddd0ade030737d2056b7b48adf8758a68ed3a43b3a41f2334cc777","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"3bff499603e7a4d81068e6457b6d94544e865fc70743b9be278f0ea515a10d86","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-rc6w6xrwc9jq/cluster-nightly-4-21-t5fxdv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c82980\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-t5fxdv: failed waiting for cluster=\"cluster-nightly-4-21-t5fxdv\" in resourcegroup=\"rg-nightly-4-21-rc6w6xrwc9jq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a7b3551-f75b-40ce-ab2a-fa119dc496ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a7b3551-f75b-40ce-ab2a-fa119dc496ea\",\n      \"name\": \"3a7b3551-f75b-40ce-ab2a-fa119dc496ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:27.933440104Z\",\n      \"endTime\": \"2026-09-28T00:26:27.662365112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-rc6w6xrwc9jq/cluster-nightly-4-21-t5fxdv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c82980\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-t5fxdv: failed waiting for cluster=\"cluster-nightly-4-21-t5fxdv\" in resourcegroup=\"rg-nightly-4-21-rc6w6xrwc9jq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a7b3551-f75b-40ce-ab2a-fa119dc496ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a7b3551-f75b-40ce-ab2a-fa119dc496ea\",\n      \"name\": \"3a7b3551-f75b-40ce-ab2a-fa119dc496ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:27.933440104Z\",\n      \"endTime\": \"2026-09-28T00:26:27.662365112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0ec8c215b0a87732d0e0d27522a14379d5fa5d22728dc33e8d4542ec9a343e32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"23dbabd9a7a25dc4e3d2c8e0a518351cb3699dffe4de727be870564d6f2f3e27","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af6ac0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-wvkkld894p88\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fcfc5cb-5f34-438a-85db-f4c1fec4ebbc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fcfc5cb-5f34-438a-85db-f4c1fec4ebbc\",\n      \"name\": \"9fcfc5cb-5f34-438a-85db-f4c1fec4ebbc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:22.544216392Z\",\n      \"endTime\": \"2026-09-27T23:34:29.497556122Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af6ac0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-wvkkld894p88\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fcfc5cb-5f34-438a-85db-f4c1fec4ebbc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fcfc5cb-5f34-438a-85db-f4c1fec4ebbc\",\n      \"name\": \"9fcfc5cb-5f34-438a-85db-f4c1fec4ebbc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:22.544216392Z\",\n      \"endTime\": \"2026-09-27T23:34:29.497556122Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"178431167bc1201a910847d3ada4bb53611081686a2823e7dcfbc16cca36d8ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"6281eff7f1eb8279cabe77f9d1f1d2478a97279bf18f2b5e7bf9104f48bb8eae","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001066e60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-vfb8zbnsnff8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8171d8b8-0ee8-41e1-9596-bb419f76fa66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8171d8b8-0ee8-41e1-9596-bb419f76fa66\",\n      \"name\": \"8171d8b8-0ee8-41e1-9596-bb419f76fa66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:39.577645259Z\",\n      \"endTime\": \"2026-09-27T23:31:39.48702037Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001066e60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-vfb8zbnsnff8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8171d8b8-0ee8-41e1-9596-bb419f76fa66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8171d8b8-0ee8-41e1-9596-bb419f76fa66\",\n      \"name\": \"8171d8b8-0ee8-41e1-9596-bb419f76fa66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:39.577645259Z\",\n      \"endTime\": \"2026-09-27T23:31:39.48702037Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1c8736b7670d5ab0daf894541280629c94e94df3c8ce5e3f7ec77f91dc0ba99b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"90c0bc090799cdf6da510003428c2c0909aad37c6a562a21a7d9711e6fac48f1","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009236a0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-nz9q7cgh9pms\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/57404065-d4e5-4294-9f65-1130a8ff568d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/57404065-d4e5-4294-9f65-1130a8ff568d\",\n      \"name\": \"57404065-d4e5-4294-9f65-1130a8ff568d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:14.225943204Z\",\n      \"endTime\": \"2026-09-28T00:27:17.671713535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009236a0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-nz9q7cgh9pms\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/57404065-d4e5-4294-9f65-1130a8ff568d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/57404065-d4e5-4294-9f65-1130a8ff568d\",\n      \"name\": \"57404065-d4e5-4294-9f65-1130a8ff568d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:14.225943204Z\",\n      \"endTime\": \"2026-09-28T00:27:17.671713535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1fc7d175b1ae592081831cfaaed526024e2568396cce7134c9da68c84cfa08ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"38536e6a87358d5db07bf788d1f69f950bd2ddf73c550c88b32453e55d15f422","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bbafa0\u003e: \n    failed to create HCP cluster np-dg-minor-ljmn55: failed waiting for cluster=\"np-dg-minor-ljmn55\" in resourcegroup=\"rg-np-dg-minor-ljmn55-vsj9pfc6qn9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43d9debf-fbb9-4fcf-9b55-9ba4bb51fc3b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43d9debf-fbb9-4fcf-9b55-9ba4bb51fc3b\",\n      \"name\": \"43d9debf-fbb9-4fcf-9b55-9ba4bb51fc3b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:45.433531245Z\",\n      \"endTime\": \"2026-09-28T00:28:47.692103938Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bbafa0\u003e: \n    failed to create HCP cluster np-dg-minor-ljmn55: failed waiting for cluster=\"np-dg-minor-ljmn55\" in resourcegroup=\"rg-np-dg-minor-ljmn55-vsj9pfc6qn9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43d9debf-fbb9-4fcf-9b55-9ba4bb51fc3b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43d9debf-fbb9-4fcf-9b55-9ba4bb51fc3b\",\n      \"name\": \"43d9debf-fbb9-4fcf-9b55-9ba4bb51fc3b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:45.433531245Z\",\n      \"endTime\": \"2026-09-28T00:28:47.692103938Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"24a83c92f91d7afd5756b31a1219c4febb0b486516019f48950be4c72eea10b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"26273269368dd302144ff0e225c732187d4fb4a38f6aeb0cfd6837ba8c834934","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-kxcpclphszg4/cluster-nightly-4-22-7c2dsj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00088c6e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-7c2dsj: failed waiting for cluster=\"cluster-nightly-4-22-7c2dsj\" in resourcegroup=\"rg-nightly-4-22-kxcpclphszg4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58004c12-b9b9-4d95-8449-fdf773362b87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58004c12-b9b9-4d95-8449-fdf773362b87\",\n      \"name\": \"58004c12-b9b9-4d95-8449-fdf773362b87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:56:56.262478975Z\",\n      \"endTime\": \"2026-09-28T00:25:57.660643706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-kxcpclphszg4/cluster-nightly-4-22-7c2dsj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00088c6e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-7c2dsj: failed waiting for cluster=\"cluster-nightly-4-22-7c2dsj\" in resourcegroup=\"rg-nightly-4-22-kxcpclphszg4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58004c12-b9b9-4d95-8449-fdf773362b87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58004c12-b9b9-4d95-8449-fdf773362b87\",\n      \"name\": \"58004c12-b9b9-4d95-8449-fdf773362b87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:56:56.262478975Z\",\n      \"endTime\": \"2026-09-28T00:25:57.660643706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"28fd20f9cf53c471c8edd6b89baedc6e66a0977ad540f78adcc7162718cacc8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"4c8ca200ae34404a7e3bd4c7adc8b9971e6a8732bacff3717a8dfe9f55b7431a","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000656de0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-slrthwkjtv9h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6ad81add-2f55-4337-958c-f7114bdff1ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6ad81add-2f55-4337-958c-f7114bdff1ed\",\n      \"name\": \"6ad81add-2f55-4337-958c-f7114bdff1ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:43.110507621Z\",\n      \"endTime\": \"2026-09-27T23:31:49.489110364Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000656de0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-slrthwkjtv9h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6ad81add-2f55-4337-958c-f7114bdff1ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6ad81add-2f55-4337-958c-f7114bdff1ed\",\n      \"name\": \"6ad81add-2f55-4337-958c-f7114bdff1ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:43.110507621Z\",\n      \"endTime\": \"2026-09-27T23:31:49.489110364Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3abee43ef45e5c32e7c75dabf901adf84c36e286495c09023464957b25b173ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"c4d86f43a9cb158593bd8be02d96713af2caf1df823e38555f25002469f3a771","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-h77pz9 with version 4.21.0-0.nightly-multi-2026-08-16-064216\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138a820\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-h77pz9: failed waiting for cluster=\"np-version-upgrade-cluster-h77pz9\" in resourcegroup=\"rg-np-version-upgrade-h77pz9-6r8s69c7447k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ba77d7f-abbf-4320-bc76-508bf6a329cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ba77d7f-abbf-4320-bc76-508bf6a329cc\",\n      \"name\": \"3ba77d7f-abbf-4320-bc76-508bf6a329cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:32.168616266Z\",\n      \"endTime\": \"2026-09-28T00:27:37.693340161Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-h77pz9 with version 4.21.0-0.nightly-multi-2026-08-16-064216\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138a820\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-h77pz9: failed waiting for cluster=\"np-version-upgrade-cluster-h77pz9\" in resourcegroup=\"rg-np-version-upgrade-h77pz9-6r8s69c7447k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ba77d7f-abbf-4320-bc76-508bf6a329cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ba77d7f-abbf-4320-bc76-508bf6a329cc\",\n      \"name\": \"3ba77d7f-abbf-4320-bc76-508bf6a329cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:32.168616266Z\",\n      \"endTime\": \"2026-09-28T00:27:37.693340161Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3c934e67be42bc1c7731ff2c1311136df4a6b597e307a00eccb451e6815d8e03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"439ff4199ab893ef47b2b24677b3a0712ca10557119868746c09e5ba33753173","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0aee0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-554wk26tw8h2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/10be9e51-c88b-4df0-a061-31b074ee852b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/10be9e51-c88b-4df0-a061-31b074ee852b\",\n      \"name\": \"10be9e51-c88b-4df0-a061-31b074ee852b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:25.519573959Z\",\n      \"endTime\": \"2026-09-27T23:33:29.494256598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0aee0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-554wk26tw8h2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/10be9e51-c88b-4df0-a061-31b074ee852b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/10be9e51-c88b-4df0-a061-31b074ee852b\",\n      \"name\": \"10be9e51-c88b-4df0-a061-31b074ee852b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:25.519573959Z\",\n      \"endTime\": \"2026-09-27T23:33:29.494256598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"54766201fc6ecc8d5d54735ea001715ff54053a40f1bba9db08c47b4395f1107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"324df14529a82b1427108ceca83d1a217a2b34761f00dac0a98207aaed8e2f6a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9e5c0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lzzgsqgdxjgf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/21bbf8c9-7a2e-43a3-bf73-bb1271c743ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/21bbf8c9-7a2e-43a3-bf73-bb1271c743ab\",\n      \"name\": \"21bbf8c9-7a2e-43a3-bf73-bb1271c743ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:09.498879782Z\",\n      \"endTime\": \"2026-09-28T00:27:17.67319034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9e5c0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lzzgsqgdxjgf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/21bbf8c9-7a2e-43a3-bf73-bb1271c743ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/21bbf8c9-7a2e-43a3-bf73-bb1271c743ab\",\n      \"name\": \"21bbf8c9-7a2e-43a3-bf73-bb1271c743ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:09.498879782Z\",\n      \"endTime\": \"2026-09-28T00:27:17.67319034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"58fd82b705aeb9c63430db03f3f97983b17d021b28ab6731b92e6689cfb675f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"c455f9856d6fa821cdf932c6b24a1d5b652aacd13e74e0b2a54394592476509f","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001280460\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-vqg75gmd6vnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e87911dd-0e87-4c8b-bdb2-aa982e1192f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e87911dd-0e87-4c8b-bdb2-aa982e1192f2\",\n      \"name\": \"e87911dd-0e87-4c8b-bdb2-aa982e1192f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.841093241Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489031363Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001280460\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-vqg75gmd6vnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e87911dd-0e87-4c8b-bdb2-aa982e1192f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e87911dd-0e87-4c8b-bdb2-aa982e1192f2\",\n      \"name\": \"e87911dd-0e87-4c8b-bdb2-aa982e1192f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.841093241Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489031363Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5cac0babba380bdf1cd0db15613c91d8ed34fcca7f2f757d623e9f75e6b2b820","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"119df034b49750d6f9010e8633d30b4bbb66194fdc1ae0c67ea3b39be3b9c543","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1ea60\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ptqx8bknktzp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4f2ab8e-cc8b-461a-8624-5179be54eabc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4f2ab8e-cc8b-461a-8624-5179be54eabc\",\n      \"name\": \"d4f2ab8e-cc8b-461a-8624-5179be54eabc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.747714732Z\",\n      \"endTime\": \"2026-09-27T23:31:59.490551228Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1ea60\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ptqx8bknktzp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4f2ab8e-cc8b-461a-8624-5179be54eabc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4f2ab8e-cc8b-461a-8624-5179be54eabc\",\n      \"name\": \"d4f2ab8e-cc8b-461a-8624-5179be54eabc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.747714732Z\",\n      \"endTime\": \"2026-09-27T23:31:59.490551228Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5cf0179163107ed7c90d23e16ef19a0e86aa478c93fe148eb9915b811151130f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"5a26c6c58f465808210315cfcafa690d409fe8a03cce1c50af1465b1cb7a8610","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f40d60\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-shvcm52drf9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01b938cc-52a2-4887-89f3-2d3cad2b72c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01b938cc-52a2-4887-89f3-2d3cad2b72c3\",\n      \"name\": \"01b938cc-52a2-4887-89f3-2d3cad2b72c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:07.744422468Z\",\n      \"endTime\": \"2026-09-28T00:27:07.672033243Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f40d60\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-shvcm52drf9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01b938cc-52a2-4887-89f3-2d3cad2b72c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01b938cc-52a2-4887-89f3-2d3cad2b72c3\",\n      \"name\": \"01b938cc-52a2-4887-89f3-2d3cad2b72c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:07.744422468Z\",\n      \"endTime\": \"2026-09-28T00:27:07.672033243Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"60f7127b46c70dc9a8a1cfc3b720fbb4e61e394ccb468a72c0903fb5a208a89b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"48061177828d591484921b302c81a59f13ae276493f33f536f8259e7bfc70589","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011eadb0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011eadb0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"65318c3474695622eac8a8de10fa848adb4d9e85a6ee30d2284e31df34d005ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"92e0a2eb8ab9de257cc1b92fbcc3e72f5a75df2a6d7cc7f2881e635a2c3b5f6d","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b98720\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vh7rhmvj6b82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82682c87-0613-4528-bbdc-b0b5471d89e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82682c87-0613-4528-bbdc-b0b5471d89e5\",\n      \"name\": \"82682c87-0613-4528-bbdc-b0b5471d89e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:47.113470008Z\",\n      \"endTime\": \"2026-09-28T00:28:47.692572721Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b98720\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vh7rhmvj6b82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82682c87-0613-4528-bbdc-b0b5471d89e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82682c87-0613-4528-bbdc-b0b5471d89e5\",\n      \"name\": \"82682c87-0613-4528-bbdc-b0b5471d89e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:47.113470008Z\",\n      \"endTime\": \"2026-09-28T00:28:47.692572721Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6745fb1ffb495ba31e2b34ddad01d9c61bae675ea2ed0a65f15bff99f11df03c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"ffb915c7a519d7b02820c9035036d88977d516b84a237b7e6e8ad2dd9b9161d8","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000908760\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-rdnft58nhhhs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6e6da606-72ef-4906-b49f-2e0d0f42ec52\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6e6da606-72ef-4906-b49f-2e0d0f42ec52\",\n      \"name\": \"6e6da606-72ef-4906-b49f-2e0d0f42ec52\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:14.609473044Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672444419Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000908760\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-rdnft58nhhhs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6e6da606-72ef-4906-b49f-2e0d0f42ec52\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6e6da606-72ef-4906-b49f-2e0d0f42ec52\",\n      \"name\": \"6e6da606-72ef-4906-b49f-2e0d0f42ec52\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:14.609473044Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672444419Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"744120443cff77332cac514914cdbd2fded1b2d21f37b2ba3a26c5ca28ac1092","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"bd0573019a5f2c7a7f36981c8e81650979ee3fa21b9ca74de2f80de6f898a00c","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f66c00\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-pbjsp2slkg85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72800f77-818e-4db0-9b79-7269b7c09db1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72800f77-818e-4db0-9b79-7269b7c09db1\",\n      \"name\": \"72800f77-818e-4db0-9b79-7269b7c09db1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.678838745Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489386628Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f66c00\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-pbjsp2slkg85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72800f77-818e-4db0-9b79-7269b7c09db1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72800f77-818e-4db0-9b79-7269b7c09db1\",\n      \"name\": \"72800f77-818e-4db0-9b79-7269b7c09db1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.678838745Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489386628Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7c7d61c26203375a3d13d5f35974a4727a77dd29a1ff6330c21c21c775c98178","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2149a88e6259e337c7dba379c74c61630a619c61b816100de91023f80eaf5155","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-ls987j\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a12660\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-ls987j: failed waiting for cluster=\"cp-ystream-upgrade-4-23-ls987j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-hstckbwcrnkj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a32dc68b-9126-427b-be57-cfe4cc635070\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a32dc68b-9126-427b-be57-cfe4cc635070\",\n      \"name\": \"a32dc68b-9126-427b-be57-cfe4cc635070\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:18.507273215Z\",\n      \"endTime\": \"2026-09-28T00:27:27.673315974Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-ls987j\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a12660\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-ls987j: failed waiting for cluster=\"cp-ystream-upgrade-4-23-ls987j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-hstckbwcrnkj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a32dc68b-9126-427b-be57-cfe4cc635070\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a32dc68b-9126-427b-be57-cfe4cc635070\",\n      \"name\": \"a32dc68b-9126-427b-be57-cfe4cc635070\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:18.507273215Z\",\n      \"endTime\": \"2026-09-28T00:27:27.673315974Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7d99f994b6b07e845fbb6c47fa7f51742ab9756e7415d9177aeae6f2fe06b785","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"e64f97e86ec58fbf99ca3a6efe60f55c9d315887853d612757b870ea0ef396c3","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001058aa0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vd4sq7v444zm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b50c03-8593-4772-9894-b52e190e1fff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b50c03-8593-4772-9894-b52e190e1fff\",\n      \"name\": \"d5b50c03-8593-4772-9894-b52e190e1fff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:49.532790064Z\",\n      \"endTime\": \"2026-09-27T23:31:49.489314628Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001058aa0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vd4sq7v444zm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b50c03-8593-4772-9894-b52e190e1fff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b50c03-8593-4772-9894-b52e190e1fff\",\n      \"name\": \"d5b50c03-8593-4772-9894-b52e190e1fff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:49.532790064Z\",\n      \"endTime\": \"2026-09-27T23:31:49.489314628Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"841a2e123b23ad6240c46e9847db69d130d3274845f64c8e7a622d8e8d1bd592","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2fe44cc5813d3d43cc5b78bad763293044af3210e10b6ee32a7f52568d9a44e7","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f66900\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-rt6qgw6rjndv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f1db31d-2da3-451a-bf69-058f9201d45c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f1db31d-2da3-451a-bf69-058f9201d45c\",\n      \"name\": \"7f1db31d-2da3-451a-bf69-058f9201d45c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:18.118119937Z\",\n      \"endTime\": \"2026-09-28T00:27:27.673012152Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f66900\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-rt6qgw6rjndv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f1db31d-2da3-451a-bf69-058f9201d45c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f1db31d-2da3-451a-bf69-058f9201d45c\",\n      \"name\": \"7f1db31d-2da3-451a-bf69-058f9201d45c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:18.118119937Z\",\n      \"endTime\": \"2026-09-28T00:27:27.673012152Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8ef056649dfb33483b9987cef098de5867af2f98019213fe0c498a4ddd2a6a6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"5ac0c3c598bb23a89a0e63b80193f5a0b989ef8f676e990362a174cef1540ecb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lhwzsr\" at install version 4.21.0-0.nightly-multi-2026-08-16-064216\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eb8de0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lhwzsr: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lhwzsr\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-728lztvgsmqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f3738ec-b0f1-4f12-8cce-11f2283bdc71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f3738ec-b0f1-4f12-8cce-11f2283bdc71\",\n      \"name\": \"8f3738ec-b0f1-4f12-8cce-11f2283bdc71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:37.862054451Z\",\n      \"endTime\": \"2026-09-28T00:26:37.695691601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lhwzsr\" at install version 4.21.0-0.nightly-multi-2026-08-16-064216\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eb8de0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lhwzsr: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lhwzsr\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-728lztvgsmqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f3738ec-b0f1-4f12-8cce-11f2283bdc71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f3738ec-b0f1-4f12-8cce-11f2283bdc71\",\n      \"name\": \"8f3738ec-b0f1-4f12-8cce-11f2283bdc71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:37.862054451Z\",\n      \"endTime\": \"2026-09-28T00:26:37.695691601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"951cf53d414b32193893c27497df89d55c07ed98af115c619a80dd3991ea437f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"1b3635689317c3e03438e1d8760fb8fbd558bfaf8f0969472e313ce9b7e51d33","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-smz8ccw67j95/cluster-nightly-5-1-n44gxt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a58f80\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-n44gxt: failed waiting for cluster=\"cluster-nightly-5-1-n44gxt\" in resourcegroup=\"rg-nightly-5-1-smz8ccw67j95\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f423c052-75d2-4bae-b4ca-4718d3554963\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f423c052-75d2-4bae-b4ca-4718d3554963\",\n      \"name\": \"f423c052-75d2-4bae-b4ca-4718d3554963\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:20.81324454Z\",\n      \"endTime\": \"2026-09-28T00:26:27.66160026Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-smz8ccw67j95/cluster-nightly-5-1-n44gxt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a58f80\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-n44gxt: failed waiting for cluster=\"cluster-nightly-5-1-n44gxt\" in resourcegroup=\"rg-nightly-5-1-smz8ccw67j95\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f423c052-75d2-4bae-b4ca-4718d3554963\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f423c052-75d2-4bae-b4ca-4718d3554963\",\n      \"name\": \"f423c052-75d2-4bae-b4ca-4718d3554963\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:20.81324454Z\",\n      \"endTime\": \"2026-09-28T00:26:27.66160026Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c79ee710511f9f520b0b0d133d3a420eb6e7043a106a744281ab5072a704a0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"062e7d036f7e68f5186271d94c1e3a10b9458a3d58b06cd0e478defd07b47e4d","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-kq44rx at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a482e0\u003e: \n    failed to create HCP cluster np-skip-minor-kq44rx: failed waiting for cluster=\"np-skip-minor-kq44rx\" in resourcegroup=\"rg-np-skip-minor-kq44rx-wrs7cdgqmf9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"name\": \"123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:09.384523535Z\",\n      \"endTime\": \"2026-09-28T00:28:17.690643945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-kq44rx at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a482e0\u003e: \n    failed to create HCP cluster np-skip-minor-kq44rx: failed waiting for cluster=\"np-skip-minor-kq44rx\" in resourcegroup=\"rg-np-skip-minor-kq44rx-wrs7cdgqmf9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"name\": \"123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:09.384523535Z\",\n      \"endTime\": \"2026-09-28T00:28:17.690643945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9e8a8eced47ea4def701575d813e8605cdacaf4a3e99f78b49824aa3356026b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"c58b99118542c64c5e8002b1a143487972b7b81409252899ec6c7d1391fd9e89","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000caaca0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-mm5xdfcrnxcg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f52e0df8-8a9a-4d7c-a040-82d256d107d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f52e0df8-8a9a-4d7c-a040-82d256d107d9\",\n      \"name\": \"f52e0df8-8a9a-4d7c-a040-82d256d107d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:13:43.929725977Z\",\n      \"endTime\": \"2026-09-28T00:42:50.716503716Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000caaca0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-mm5xdfcrnxcg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f52e0df8-8a9a-4d7c-a040-82d256d107d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f52e0df8-8a9a-4d7c-a040-82d256d107d9\",\n      \"name\": \"f52e0df8-8a9a-4d7c-a040-82d256d107d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:13:43.929725977Z\",\n      \"endTime\": \"2026-09-28T00:42:50.716503716Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a56bfd68de82d6be4379e6bfb13e7d171fa61d6bab3579f0518203f0de6b3a77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"8f18f1765cdb58fe6bdd47ab7382897cef64b6ff0d2819e45a6edfb388427d67","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b67200\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-697wjx7zzzdm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43b94205-fe8d-47b3-a4a4-7e1266efc4c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43b94205-fe8d-47b3-a4a4-7e1266efc4c0\",\n      \"name\": \"43b94205-fe8d-47b3-a4a4-7e1266efc4c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:15.695573565Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672891195Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b67200\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-697wjx7zzzdm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43b94205-fe8d-47b3-a4a4-7e1266efc4c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43b94205-fe8d-47b3-a4a4-7e1266efc4c0\",\n      \"name\": \"43b94205-fe8d-47b3-a4a4-7e1266efc4c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:15.695573565Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672891195Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a72add00392e5ef10dc09dedaeeb614333748d706a8cfbd2ab583f2feca5b670","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2b107b242908c25a7edb0c1f9ed1b123d45886939e1691d3ccbe4936336d4c1a","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db4b20\u003e: \n    failed to create HCP cluster ea-listdnk44k: failed waiting for cluster=\"ea-listdnk44k\" in resourcegroup=\"ea-list-t5csnm9zbqrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0c6958a-5bab-435e-948c-a6226195419e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0c6958a-5bab-435e-948c-a6226195419e\",\n      \"name\": \"e0c6958a-5bab-435e-948c-a6226195419e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:08.01833551Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672737432Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db4b20\u003e: \n    failed to create HCP cluster ea-listdnk44k: failed waiting for cluster=\"ea-listdnk44k\" in resourcegroup=\"ea-list-t5csnm9zbqrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0c6958a-5bab-435e-948c-a6226195419e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0c6958a-5bab-435e-948c-a6226195419e\",\n      \"name\": \"e0c6958a-5bab-435e-948c-a6226195419e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:08.01833551Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672737432Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aa09450f4b943aef4dbb786bd5c0453be3ce0ee42073f77c6dcadd13bcc4ff51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"97240d8885a0bdfc86afa503aac43fee59507e22b5387aeb628e8d3e06faee04","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"acef77d4b3c6fc57440f5e8d65c5680fa58e023131a63dbe3dc36aae76413810","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"a7a62f4ae83f39cc2aa5610a1fa39913dbb89c8a7e136bf0b375398a36a4cbe5","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc520\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-47d827vvrw49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb385e97-6ba2-4c01-9779-4cf02866139b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb385e97-6ba2-4c01-9779-4cf02866139b\",\n      \"name\": \"eb385e97-6ba2-4c01-9779-4cf02866139b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:06.639027634Z\",\n      \"endTime\": \"2026-09-27T23:34:09.49706636Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc520\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-47d827vvrw49\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb385e97-6ba2-4c01-9779-4cf02866139b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb385e97-6ba2-4c01-9779-4cf02866139b\",\n      \"name\": \"eb385e97-6ba2-4c01-9779-4cf02866139b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:05:06.639027634Z\",\n      \"endTime\": \"2026-09-27T23:34:09.49706636Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b15495eee01eb29f0cd0717ce6d81e04ccd69da3e1a61ce499cbd914569714c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2fe43e4aefe8fad2aa3b31f0b353a06e9223a44a64d2a9d195a8c3bd94ddf6e8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lc2wg\" for cluster \"cluster-nightly-4-20-7lc2wg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de6640\u003e: \n    failed to create NodePool np-7lc2wg: failed starting nodepool creation \"np-7lc2wg\" for cluster \"cluster-nightly-4-20-7lc2wg\" in resourcegroup=\"rg-nightly-4-20-7g2xnmzkrznn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-7g2xnmzkrznn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-7lc2wg/nodePools/np-7lc2wg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lc2wg\" for cluster \"cluster-nightly-4-20-7lc2wg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de6640\u003e: \n    failed to create NodePool np-7lc2wg: failed starting nodepool creation \"np-7lc2wg\" for cluster \"cluster-nightly-4-20-7lc2wg\" in resourcegroup=\"rg-nightly-4-20-7g2xnmzkrznn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-7g2xnmzkrznn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-7lc2wg/nodePools/np-7lc2wg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b723363a0ec2d14c8c7aa1df3d43a711864b361dede55b7ef45491afd980740c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"2ac5f8dbb6cec6887c383dfebee159a130fca9941d88962c9ad1e6a8cd766041","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af4a40\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-z9xpg6dx44vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb3bb885-d121-4d31-94cb-2c2e50676335\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb3bb885-d121-4d31-94cb-2c2e50676335\",\n      \"name\": \"eb3bb885-d121-4d31-94cb-2c2e50676335\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:43.933442761Z\",\n      \"endTime\": \"2026-09-27T23:31:49.489929931Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af4a40\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-z9xpg6dx44vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb3bb885-d121-4d31-94cb-2c2e50676335\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb3bb885-d121-4d31-94cb-2c2e50676335\",\n      \"name\": \"eb3bb885-d121-4d31-94cb-2c2e50676335\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:43.933442761Z\",\n      \"endTime\": \"2026-09-27T23:31:49.489929931Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"be09a76a6e5dace9d5ae0f1a299aa646d37897efc3200260c57060fff8fe4379","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"94c267ae0b903b82dad324aa6a1aa031b444202e67e55ac10746b4caf0554318","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0016b0ae0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fvm5hjf9trrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd4fad33-ebbb-47e5-94e2-f6a999a393f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd4fad33-ebbb-47e5-94e2-f6a999a393f0\",\n      \"name\": \"bd4fad33-ebbb-47e5-94e2-f6a999a393f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:13.954217314Z\",\n      \"endTime\": \"2026-09-28T00:27:17.671690658Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0016b0ae0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fvm5hjf9trrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd4fad33-ebbb-47e5-94e2-f6a999a393f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd4fad33-ebbb-47e5-94e2-f6a999a393f0\",\n      \"name\": \"bd4fad33-ebbb-47e5-94e2-f6a999a393f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:13.954217314Z\",\n      \"endTime\": \"2026-09-28T00:27:17.671690658Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c8c7302b81a87f72220d0f5ac848ff42067157fc13f6a1e9c256f451d191eb82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"43fdd0f3790007d3ecddd7c1c777edb218e066a82db7780cdfade1af4e52fdd8","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000578a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xq6nmlhdcd84\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc8edfd8-8b25-47bc-9569-bf80fa4decac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc8edfd8-8b25-47bc-9569-bf80fa4decac\",\n      \"name\": \"bc8edfd8-8b25-47bc-9569-bf80fa4decac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.158928262Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489753067Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000578a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xq6nmlhdcd84\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc8edfd8-8b25-47bc-9569-bf80fa4decac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc8edfd8-8b25-47bc-9569-bf80fa4decac\",\n      \"name\": \"bc8edfd8-8b25-47bc-9569-bf80fa4decac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.158928262Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489753067Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cf9d490e066d7695ee14b811d49ce4bdc74ccd28d313768c152e638fcc556ca3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"11247ce48003ac7a491f84d736813861aaf0b3430f2681f5c7d7b51f7e3584ce","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075dac0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bk8w68gw94vn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d275dc6f-a5af-4a23-9d0b-63cfeda20389\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d275dc6f-a5af-4a23-9d0b-63cfeda20389\",\n      \"name\": \"d275dc6f-a5af-4a23-9d0b-63cfeda20389\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:45.557708779Z\",\n      \"endTime\": \"2026-09-27T23:32:49.493110089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075dac0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bk8w68gw94vn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d275dc6f-a5af-4a23-9d0b-63cfeda20389\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d275dc6f-a5af-4a23-9d0b-63cfeda20389\",\n      \"name\": \"d275dc6f-a5af-4a23-9d0b-63cfeda20389\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:45.557708779Z\",\n      \"endTime\": \"2026-09-27T23:32:49.493110089Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dec8ca0990481e60f758663da91603a28b832a513c340401376e700d6f189226","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"408ef8388aaa44003433347eae7dd6c7a856b7078ffdd8abfc9a831c782bf6d6","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd09a0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-twf8pdtc8q4r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d537e3b3-f194-4f0f-8c18-2922a0c2c1cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d537e3b3-f194-4f0f-8c18-2922a0c2c1cb\",\n      \"name\": \"d537e3b3-f194-4f0f-8c18-2922a0c2c1cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:49.901495557Z\",\n      \"endTime\": \"2026-09-27T23:31:49.488818493Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd09a0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-twf8pdtc8q4r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d537e3b3-f194-4f0f-8c18-2922a0c2c1cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d537e3b3-f194-4f0f-8c18-2922a0c2c1cb\",\n      \"name\": \"d537e3b3-f194-4f0f-8c18-2922a0c2c1cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:49.901495557Z\",\n      \"endTime\": \"2026-09-27T23:31:49.488818493Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e00c0014bbf5bc9aa6d200dd578ac0cfc8baa1587cea0cebb48d05e988f72cf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"9d3c0ca51b7611f46d699909444e88f85d187197f0007cf922f659d4688bf7f5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001257740\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001257740\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"e1e519b0d2de8735e989f7c3d6f6aab0ac06e3e5c3a96b5c5849d3d095a12ad9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"f8cd0e057dfd4f130119ba94d650784ae8733e232829e5be0bc8cac32b386df0","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000733e40\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-85nsqn7c57p8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d1c34fa-c57c-4cba-b433-80703e25ab62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d1c34fa-c57c-4cba-b433-80703e25ab62\",\n      \"name\": \"4d1c34fa-c57c-4cba-b433-80703e25ab62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.077800351Z\",\n      \"endTime\": \"2026-09-27T23:31:59.49007937Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000733e40\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-85nsqn7c57p8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d1c34fa-c57c-4cba-b433-80703e25ab62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d1c34fa-c57c-4cba-b433-80703e25ab62\",\n      \"name\": \"4d1c34fa-c57c-4cba-b433-80703e25ab62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.077800351Z\",\n      \"endTime\": \"2026-09-27T23:31:59.49007937Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e3abd7b579b8b46f8d4ece19cf09cde71bb632f303e73c06898ededa4e6ee287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"1813a0655895bafdd0fa7795ce7106f0e053c012c9ed0db0b61b9a3da768370c","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004df1e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-7fp82sngxtbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7d3c6b5b-9c7f-4734-b248-2bfbc789fea0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7d3c6b5b-9c7f-4734-b248-2bfbc789fea0\",\n      \"name\": \"7d3c6b5b-9c7f-4734-b248-2bfbc789fea0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:14.235161565Z\",\n      \"endTime\": \"2026-09-28T00:27:17.673483272Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004df1e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-7fp82sngxtbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7d3c6b5b-9c7f-4734-b248-2bfbc789fea0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7d3c6b5b-9c7f-4734-b248-2bfbc789fea0\",\n      \"name\": \"7d3c6b5b-9c7f-4734-b248-2bfbc789fea0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:14.235161565Z\",\n      \"endTime\": \"2026-09-28T00:27:17.673483272Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e5ea70d568bf661c8b27905c3a8d06e9fec1f452ef6e4627ce0d26a020639af3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"0d62eb4a49f95b17a29f90933dfbd4d485316c175eb1797cd4f2382596e3a05e","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02800\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ss8z8mq428kv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"name\": \"5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.434225001Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489114255Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02800\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ss8z8mq428kv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"name\": \"5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.434225001Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489114255Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ee0105a5111a17802903ea7c69f5281cea1329cf7aa9a637003d70ca80085027","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"6e4f2cee72f249ad623ac333e98b4d5b46ed800feced1b98e734aad6426d4f57","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000816cc0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pbxqtq9q975p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/584ba00c-67ec-4958-af2b-1eadce8027d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/584ba00c-67ec-4958-af2b-1eadce8027d4\",\n      \"name\": \"584ba00c-67ec-4958-af2b-1eadce8027d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:16.758777697Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672510334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000816cc0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pbxqtq9q975p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/584ba00c-67ec-4958-af2b-1eadce8027d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/584ba00c-67ec-4958-af2b-1eadce8027d4\",\n      \"name\": \"584ba00c-67ec-4958-af2b-1eadce8027d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:16.758777697Z\",\n      \"endTime\": \"2026-09-28T00:27:17.672510334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"efcf5aea6b877d6bf611b02c0b622927cb0fc14d7638922357d77a21b9b9041b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"573441d01bc3a5dc38836ea38903d87ee6c2f25b6c4d93a979010af06a1058f4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-62gr7l\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cab380\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-62gr7l: failed waiting for cluster=\"cp-ystream-upgrade-5-0-62gr7l\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-cswjlcdrpdmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/feaf6284-b05a-47b3-9cdc-9b544c41a1ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/feaf6284-b05a-47b3-9cdc-9b544c41a1ca\",\n      \"name\": \"feaf6284-b05a-47b3-9cdc-9b544c41a1ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:16.133688106Z\",\n      \"endTime\": \"2026-09-28T00:27:17.673536863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-62gr7l\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cab380\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-62gr7l: failed waiting for cluster=\"cp-ystream-upgrade-5-0-62gr7l\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-cswjlcdrpdmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/feaf6284-b05a-47b3-9cdc-9b544c41a1ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/feaf6284-b05a-47b3-9cdc-9b544c41a1ca\",\n      \"name\": \"feaf6284-b05a-47b3-9cdc-9b544c41a1ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:58:16.133688106Z\",\n      \"endTime\": \"2026-09-28T00:27:17.673536863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f22d1e9d7cd33f909f1359266714d5354fc040450e901d15bf749fd2b1086546","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"85b4ba4b35a851104183637b8ccb8be950663bcfce1868008fe118cbe84ca8eb","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010cc6e0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kpx28dg62cnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a15c63c7-6d52-4412-81db-0529880a3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a15c63c7-6d52-4412-81db-0529880a3e86\",\n      \"name\": \"a15c63c7-6d52-4412-81db-0529880a3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:51.012142821Z\",\n      \"endTime\": \"2026-09-27T23:31:59.490175937Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010cc6e0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kpx28dg62cnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a15c63c7-6d52-4412-81db-0529880a3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a15c63c7-6d52-4412-81db-0529880a3e86\",\n      \"name\": \"a15c63c7-6d52-4412-81db-0529880a3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:51.012142821Z\",\n      \"endTime\": \"2026-09-27T23:31:59.490175937Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f5476d4f09ba5293273c2db9cb9e9e7fd1ddda8c2065cef7af3fc2b5ae8ca784","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"898be32d8771fbe17057829ab379f352c641969b8778d5c746d951682484e908","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fde840\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/449c12fa-ccaf-4ac5-ac25-394845c24b25\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/449c12fa-ccaf-4ac5-ac25-394845c24b25\",\n      \"name\": \"449c12fa-ccaf-4ac5-ac25-394845c24b25\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:30.661590969Z\",\n      \"endTime\": \"2026-09-27T23:32:39.492855664Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fde840\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/449c12fa-ccaf-4ac5-ac25-394845c24b25\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/449c12fa-ccaf-4ac5-ac25-394845c24b25\",\n      \"name\": \"449c12fa-ccaf-4ac5-ac25-394845c24b25\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:03:30.661590969Z\",\n      \"endTime\": \"2026-09-27T23:32:39.492855664Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fc114153404b36d38e4ce76f0068922976fa945c16621b4a25da3583eb19f556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"ff4a3c1c2ea8ddac1bb3f1cf2f714cd7e5290bccd6c0f8a4734033754899293e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062c960\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-64wkhbmzb7g8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3fd835f-0f6e-4edc-8259-f4ba7eb616f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3fd835f-0f6e-4edc-8259-f4ba7eb616f9\",\n      \"name\": \"a3fd835f-0f6e-4edc-8259-f4ba7eb616f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:10.792622349Z\",\n      \"endTime\": \"2026-09-27T23:33:19.494656442Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062c960\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-64wkhbmzb7g8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3fd835f-0f6e-4edc-8259-f4ba7eb616f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3fd835f-0f6e-4edc-8259-f4ba7eb616f9\",\n      \"name\": \"a3fd835f-0f6e-4edc-8259-f4ba7eb616f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:04:10.792622349Z\",\n      \"endTime\": \"2026-09-27T23:33:19.494656442Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fe5477fe50577c54651a86f5a8b9bab60f456c4b2e0ac65b2463fb9bb99e020f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","failed_at":"e2e","signature_id":"96595f560f0d4e5f6272ce653d963aae178a27545a8e2d6bb342759a707380d5","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9ca20\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-r8slnk5nqrjs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7fb1b7c5-3455-4ff7-a7c1-0a6068710299\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7fb1b7c5-3455-4ff7-a7c1-0a6068710299\",\n      \"name\": \"7fb1b7c5-3455-4ff7-a7c1-0a6068710299\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:51.892589675Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489602625Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9ca20\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-r8slnk5nqrjs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7fb1b7c5-3455-4ff7-a7c1-0a6068710299\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7fb1b7c5-3455-4ff7-a7c1-0a6068710299\",\n      \"name\": \"7fb1b7c5-3455-4ff7-a7c1-0a6068710299\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:51.892589675Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489602625Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":46,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","b0e52789a9254af02bf84a6572690557e6e470276175202a83ecb44c6d1cf3d1","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f"],"matched_occurrences":46,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2104028630299447297","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-27T02:00:24Z","started_at":"2026-09-27T02:00:24Z","completed_at":"2026-09-27T03:49:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T04:02:21.137469865Z","timing_metadata_checked_at":"2026-09-27T04:02:21.137469865Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T04:02:21.140768253Z","region_metadata_checked_at":"2026-09-27T04:02:21.140768253Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"82a56c156767ac2d6b36d8e98df95501e7e00b6e0b148877509703abf481ce46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2104028630299447297","occurred_at":"2026-09-27T02:00:24Z","failed_at":"e2e","signature_id":"19018416e82eb03f9b16613ac5e37d0f324be8fff878665862029a9f827bc1f3","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996660\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z624hhq7js4c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"name\": \"c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:15:40.256886606Z\",\n      \"endTime\": \"2026-09-27T02:34:46.354762492Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996660\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z624hhq7js4c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"name\": \"c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:15:40.256886606Z\",\n      \"endTime\": \"2026-09-27T02:34:46.354762492Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}