{"meta":{"date":"2026-09-18","timezone":"UTC","generated_at":"2026-10-31T08:34:28Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":15,"failed_runs":9,"runs_with_occurrences":9,"runs_with_failure_pattern_matches":9,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T23:00:32Z","started_at":"2026-09-18T23:00:32Z","completed_at":"2026-09-19T00:58:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:53.003406006Z","timing_metadata_checked_at":"2026-09-19T03:30:53.003406006Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:53.006325117Z","region_metadata_checked_at":"2026-09-19T03:30:53.006325117Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"259bba84953ec64fd500f1e1cbc63c042bdfbf7e6ef233c9e412c89782c9d5ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"a1f4334212b041c83192bb8549659dd10bbe7f9e8bd239e3cb44f42460834cd6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00114d9e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zgncvtrh7b9x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00114d9e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zgncvtrh7b9x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"280e515396f186e3886a96168105237fdeb9a861ba05abb2a02a44494156bd1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"31f227f2999d078a72461d40e7b0f7d39a61c55e78688c72750e61796cea0c0e","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00055c870\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00055c870\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"424d51ffdd8c2fc3c8452117a476bb9c18f33b5ad1967acfb945ce3cc6ad1e0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"6b756bbe3882dd3ad1f64906dbb7701108fcd8092065edc0fdfc339e155785c9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5fc5a39273ae3815079ff600ab37e72e8590f6e9e89c192c12293185d283ca71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"d656f1256e1df9e299a9ee99092b293d1d1b5f5763759abb016ae0c94e36cbee","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b80720\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-nz9bqqgzphzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b80720\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-nz9bqqgzphzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"815f7955493dcfc7a0975aebbdae33ea9d84236a65d7d5081452ba1b848af1f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"4e7909db88252003612c7eb675142841e8737860b4601bc7ba1c1c1147d59585","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"88ae6b72fd1057b68a25a91f5d74933c4ed87683bd8c641ef5832bfc4f83f4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"e69d26f721f1d960b9d056d864133ef61ba83a27918de693c75cfc743d5fc4a8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001376700\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001376700\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded"}},{"row_id":"c0726cc103e440c504cb6cb2769843c5ae0d5d03a70c2026974c8b0dcf3073f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"19958c8a5897194ef3182c6ab16be4ac9029906aade55bb4256e8a9bbd189d3c","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ceb32b2f6ab8ba50124aad0994ad3c53e51b427ca7d9d87166b3dd55ebcb685f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"9ef798cef3d572a0e66143f21817eeeceae70fa379c7c12f948b17be196688fc","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002804f90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002804f90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d3581da71ac75f1a969ab2c4b2cefed0e13faba11e644dca136a9992982e31f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"491628b312668f06a0a9bf2464e7de82cea05fd0fafcb43e52e269cbc904606b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011be60\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011be60\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d38c28b28de215f24af3b95d1ebf5cb3dd3de972ab46f0d814d3175b8e5fb9a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"8807298fe1e8269d8ba1826e326fe8888ecc37b86d4680f87d9ef650960b1b45","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d7cb3166677cce2b36068777204e054cddad64fdcc620110ae7e7c727911cf22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"889f4b2cac9cd859f4a4017cf24ccfb20aa30ab6ad074847bdec286491246277","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00160a220\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00160a220\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f0d86640b3e51bc4e6e403a2c7ef8cadae28adc5d5b116b53a9d4592784027c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","failed_at":"e2e","signature_id":"b07971657c34858950043966384bf3450b2bd8793b71995b336ae5e99ce5d756","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98aa0\u003e: \n failed to create NodePool np-xrmv7w: failed starting nodepool creation \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\" in resourcegroup=\"rg-nightly-4-20-xx4pb26cwd2j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xx4pb26cwd2j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-xrmv7w/nodePools/np-xrmv7w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98aa0\u003e: \n failed to create NodePool np-xrmv7w: failed starting nodepool creation \"np-xrmv7w\" for cluster \"cluster-nightly-4-20-xrmv7w\" in resourcegroup=\"rg-nightly-4-20-xx4pb26cwd2j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xx4pb26cwd2j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-xrmv7w/nodePools/np-xrmv7w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101056312820371456","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T21:10:35Z","started_at":"2026-09-18T21:10:35Z","completed_at":"2026-09-18T23:43:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:52.726416092Z","timing_metadata_checked_at":"2026-09-19T03:30:52.726416092Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:52.730859608Z","region_metadata_checked_at":"2026-09-19T03:30:52.730859608Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T17:06:01Z","started_at":"2026-09-18T17:06:01Z","completed_at":"2026-09-18T20:17:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:52.608720461Z","timing_metadata_checked_at":"2026-09-19T03:30:52.608720461Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:52.612145174Z","region_metadata_checked_at":"2026-09-19T03:30:52.612145174Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"871c8ec7c1bb43b1a04353558130aef18c8efda71d9f41783a2527f39b69e118","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","occurred_at":"2026-09-18T17:06:01Z","failed_at":"e2e","signature_id":"e22784c3104fac10937a42e7b5069677dd3f893046b62e795e223bfab93b9fcd","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003578876366848","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T17:05:15Z","started_at":"2026-09-18T17:05:15Z","completed_at":"2026-09-18T20:38:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:52.112652046Z","timing_metadata_checked_at":"2026-09-19T03:30:52.112652046Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:52.115847857Z","region_metadata_checked_at":"2026-09-19T03:30:52.115847857Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T17:02:57Z","started_at":"2026-09-18T17:02:57Z","completed_at":"2026-09-18T20:52:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-19T03:30:51.889272128Z","timing_metadata_checked_at":"2026-09-19T03:30:51.889272128Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-19T03:30:51.892381939Z","region_metadata_checked_at":"2026-09-19T03:30:51.892381939Z"},"failed_at":["e2e"],"failed_test_count":54,"occurrences":[{"row_id":"0211e8e14e2a5b3d4f239b1c4fda1a84aad41a8b73b2fead3f2598f9478c865b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"08737fbd1b615be947880829b89bc4c3a56995f666d4512eff62626a308dd556","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"026d3b03aa916596f1a1e080a00d6204278080d06f2daa4c9aa07de7edf023e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"5bd085aa7d4e7efc364c83356bec393db7595ff945fba164d18f342fedc7d074","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a60580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65kllg7cv2w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"name\": \"e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:07.854642874Z\",\n \"endTime\": \"2026-09-18T18:35:13.922823026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a60580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65kllg7cv2w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"name\": \"e3de0bda-9b1d-4eb0-bf9e-927086d1d405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:07.854642874Z\",\n \"endTime\": \"2026-09-18T18:35:13.922823026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"028ded3d1d349d230f3014d3133becc8ec61fc7737e04f65d574de6386d0e390","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ed3aaafec71e925920ce96600e6accd52368d8c067bd7c98fb0f3dd54c8990b6","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-w7lg9w\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017e2320\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-w7lg9w: failed waiting for cluster=\"cp-ystream-upgrade-5-0-w7lg9w\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-whnjtcclp2gm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"name\": \"58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:40.681093019Z\",\n \"endTime\": \"2026-09-18T19:30:40.290116436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-w7lg9w\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017e2320\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-w7lg9w: failed waiting for cluster=\"cp-ystream-upgrade-5-0-w7lg9w\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-whnjtcclp2gm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"name\": \"58ae58c3-0f2b-4f24-99ef-9d3487e0c74e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:40.681093019Z\",\n \"endTime\": \"2026-09-18T19:30:40.290116436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0373f8fa5fb1bd7d61cc75a57fc3b50213c59631360ac5d4f8a1572a4e47d5b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"087e1145946b4175b23e226d2c422f8fbe6adcface041612342c65176f135291","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"155ede202b1081984b4c410c7ba51984193ce8148d01068e8d896ff5ef665dbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"279014bba3c91c8c1147706aa208a7ca8dab431b40a05ae4dc2507ffe4bd82d2","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc600\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-nhxrjzhdkw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"name\": \"954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:57.113094659Z\",\n \"endTime\": \"2026-09-18T20:23:57.027116398Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abc600\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-nhxrjzhdkw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"name\": \"954f356d-f569-48c9-8a16-ac8774f18b21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:57.113094659Z\",\n \"endTime\": \"2026-09-18T20:23:57.027116398Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"181d3dc35bc76068b616a3506fad1a33714c6daa6635fe953004e37b22d37cb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"020177279fd83a306292e61e3b6eb559ce1bd5ba1e079600d6fe28da054a781b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1edc7ae5a00e1a19fc62643eb83c9ed810818f27e5c0e354f0af6043253c1f1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"7d0f6a2cda1c21acb4480d38818269f535fbffb2494dac6f98b849200db6bc5d","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ef160\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-v6jpgxxsfw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"name\": \"c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:46.580974029Z\",\n \"endTime\": \"2026-09-18T18:33:53.917669415Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ef160\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-v6jpgxxsfw4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"name\": \"c804e049-d95a-4f1c-8a94-12089c6d4c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:46.580974029Z\",\n \"endTime\": \"2026-09-18T18:33:53.917669415Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"31c14810e54846bacdfc2e4432cab2fbadbbd703197294a85045eb38729d32f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"b971663cd5f630a57997cfad6d7efdb8293b9c2c40b6d6ab7621acde7a5c5511","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-v8gljqqvz6w4/cluster-candidate-4-22-ts6cw9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013bebc0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ts6cw9: failed waiting for cluster=\"cluster-candidate-4-22-ts6cw9\" in resourcegroup=\"rg-candidate-4-22-v8gljqqvz6w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"name\": \"6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:17.002894711Z\",\n \"endTime\": \"2026-09-18T19:29:20.288780423Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-v8gljqqvz6w4/cluster-candidate-4-22-ts6cw9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013bebc0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ts6cw9: failed waiting for cluster=\"cluster-candidate-4-22-ts6cw9\" in resourcegroup=\"rg-candidate-4-22-v8gljqqvz6w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"name\": \"6ecb16b4-c24f-4db6-986c-dc7d0a058ebe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:17.002894711Z\",\n \"endTime\": \"2026-09-18T19:29:20.288780423Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"331e88311fd0ea4b998a3fdc10cdbfb01eba5a8f61c53f1db7e070c88fc52d6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"bec4f85eb91db9f106f1504bbcfbd5d619f318abcff9dccca5f00cd3af48e378","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-hs7c8h\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001204dc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-hs7c8h: failed waiting for cluster=\"cluster-zstream-4-20-hs7c8h\" in resourcegroup=\"rg-zstream-upgrade-4-20-4v6msf6jsvhm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"name\": \"fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:45.232517978Z\",\n \"endTime\": \"2026-09-18T19:29:50.287676961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-hs7c8h\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001204dc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-hs7c8h: failed waiting for cluster=\"cluster-zstream-4-20-hs7c8h\" in resourcegroup=\"rg-zstream-upgrade-4-20-4v6msf6jsvhm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"name\": \"fe569f4a-6146-4ebf-90e6-4fef2b13a008\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:45.232517978Z\",\n \"endTime\": \"2026-09-18T19:29:50.287676961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37d6c057da66b2131f940542ca522868cfd86f188b450106747c480a75abe258","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"bfbf541997c9dcf2457e39f6d9a3f3675b64df774c7c81e642e24bee8b14665a","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae2e60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-92vpld7m4q55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"name\": \"043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.023266499Z\",\n \"endTime\": \"2026-09-18T20:24:27.026850238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae2e60\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-92vpld7m4q55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"name\": \"043f530f-1cb5-4b43-ae85-55bc6d808229\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.023266499Z\",\n \"endTime\": \"2026-09-18T20:24:27.026850238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37f66f6788749252764b0a8c2c2f1bda1581ac4c5605743f5364bdc2196fcde7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"60ea4497c6186b3e803d7e6ac044255c8c915fd98b1ff264e2058f0d46b0df0e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded"}},{"row_id":"46490c8efaf59582c890fd95c754902cffd4813cff9e0645e96637af76d19cb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"48193ab97021b80941212fe6bcda29d0c742e88d6b74bd22b1e5757b7e1b3bbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"88f393269cef98e4be13d3c8ab69c7daf8948bb76a2277ef306df923eea104ad","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000872e40\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9pzlkssjh8rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"name\": \"32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:49.139622647Z\",\n \"endTime\": \"2026-09-18T19:28:50.286578702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000872e40\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9pzlkssjh8rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"name\": \"32dd996c-52a3-42f2-90b1-59c3004ebd0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:49.139622647Z\",\n \"endTime\": \"2026-09-18T19:28:50.286578702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d1e66fffc20ec5e58eecd30aebd40c1a11ebb910dd534010855e07e3b8c59d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"d992a344dec95549926180b213136a61987bad3b2b2cfe5a8cedabfb0cba64d2","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000730dc0\u003e: \n failed to create HCP cluster ea-listht4pz6: failed waiting for cluster=\"ea-listht4pz6\" in resourcegroup=\"ea-list-zfzjn95sp6z4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"name\": \"1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:22.470708216Z\",\n \"endTime\": \"2026-09-18T19:31:30.292183292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000730dc0\u003e: \n failed to create HCP cluster ea-listht4pz6: failed waiting for cluster=\"ea-listht4pz6\" in resourcegroup=\"ea-list-zfzjn95sp6z4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"name\": \"1e50a71b-2480-4fde-a3f4-65f1ffbe4dbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:22.470708216Z\",\n \"endTime\": \"2026-09-18T19:31:30.292183292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4fd60e4df1936e8425e6757a96602bd1361a328d7fa10f97e72f1f1f7973dfde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"8c17a49c3b1087a5252885b16d63b8887ba96a24c838b70dae74c83a59595562","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e460\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gq5kj4t7v9lj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"name\": \"33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:54.936239942Z\",\n \"endTime\": \"2026-09-18T18:35:03.923586817Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e460\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gq5kj4t7v9lj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"name\": \"33d2aa70-41e7-4e37-bdad-c413801367c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:54.936239942Z\",\n \"endTime\": \"2026-09-18T18:35:03.923586817Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5394b36300091ca0c568cba92d1ad99ffca033e0f84d929b305fd4d4cf235239","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"23eff0283d14160643a2c1b951b106f62c524eb771d25036e9a90406a0924c53","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e7760\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-krl8lpfgmlzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"name\": \"4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.707548645Z\",\n \"endTime\": \"2026-09-18T18:33:33.918055388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e7760\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-krl8lpfgmlzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"name\": \"4dc5125d-f2b4-4bbf-8e4e-933cde635436\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.707548645Z\",\n \"endTime\": \"2026-09-18T18:33:33.918055388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"58323156d12d6e1d9e17febc1c8b442b6542dca5f880dc95ebb22d804fd4eb44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"cb28d0b51212f011a6a9a6a3530556dc58c4a4d691d64d362e422cfc091e42ee","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077bdc0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-tsw82p5n5s4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"name\": \"5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:11.953070547Z\",\n \"endTime\": \"2026-09-18T19:31:20.290227685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077bdc0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-tsw82p5n5s4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"name\": \"5ff71734-3c8e-483d-9262-a12ff7b3f337\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:11.953070547Z\",\n \"endTime\": \"2026-09-18T19:31:20.290227685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"607499447b8a29de495eb18348a9a9d13e0639eea66ced853e22cfe011e86743","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"18459d5e026c91320fea1e6f4619d65e0f3c2cce677a7bf1890dbe9529aa82dd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vnd6pm4rt5md/cluster-candidate-4-21-gkgzcb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ec140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gkgzcb: failed waiting for cluster=\"cluster-candidate-4-21-gkgzcb\" in resourcegroup=\"rg-candidate-4-21-vnd6pm4rt5md\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"name\": \"fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.815434003Z\",\n \"endTime\": \"2026-09-18T19:29:20.286956387Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vnd6pm4rt5md/cluster-candidate-4-21-gkgzcb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ec140\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gkgzcb: failed waiting for cluster=\"cluster-candidate-4-21-gkgzcb\" in resourcegroup=\"rg-candidate-4-21-vnd6pm4rt5md\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"name\": \"fefb2d3d-5d17-4ba0-9f25-ca66939b6a12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.815434003Z\",\n \"endTime\": \"2026-09-18T19:29:20.286956387Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"621b83b49d4450c11472cfce62b9a69bfaf21557764758dd59b3472d160e4364","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"4d196dbdd6adc5cbd0b42174989b35c61699ec802a9b43e61050ca43a4a526db","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2c500\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-djhd95nkq4ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"name\": \"f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:48.39283405Z\",\n \"endTime\": \"2026-09-18T20:26:57.037154264Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2c500\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-djhd95nkq4ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"name\": \"f1f0eeaa-4da5-4317-9297-5a813aab6f33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:48.39283405Z\",\n \"endTime\": \"2026-09-18T20:26:57.037154264Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"691e134cb1e2fe5b7b0440dfbd28efa2a597a50cae196ce8be46b9c1a5bb18d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"f6a9a5ad677f486ebe2e2116b48d59eff2dab4dc02f9785191be923be7559549","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001314da0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rgw6mwjd2jdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"name\": \"09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:35.503440276Z\",\n \"endTime\": \"2026-09-18T18:33:43.918161334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001314da0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rgw6mwjd2jdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"name\": \"09ad358c-d8b2-4b77-b6e1-79ace9bdc096\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:35.503440276Z\",\n \"endTime\": \"2026-09-18T18:33:43.918161334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ac49c323b67c7d7110ee9bd50fbf775e842e6aecd22499e02812121d7a59453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"a086d8d16ebbda1f55085340c84766813ece581729f4e4db8f61f9c0b1339c8e","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010724a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-fp6bsqq92kn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"name\": \"55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.41327514Z\",\n \"endTime\": \"2026-09-18T18:33:23.916940698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010724a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-fp6bsqq92kn8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"name\": \"55377e80-bcab-47a5-9c3e-4350a26d1727\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.41327514Z\",\n \"endTime\": \"2026-09-18T18:33:23.916940698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c9e225debfb06a93ce8fd4a4082d4065a29742d6d896646708869494b7fb8aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"807a30302c1afe55628a0e91c1d8d2d1cf846d02300aeafd38841fdc5629959a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-cjg6sk\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ed60\u003e: \n failed to create HCP cluster cluster-zstream-4-21-cjg6sk: failed waiting for cluster=\"cluster-zstream-4-21-cjg6sk\" in resourcegroup=\"rg-zstream-upgrade-4-21-h8qwfrzsn2zp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"name\": \"e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:38.211424873Z\",\n \"endTime\": \"2026-09-18T19:29:40.287386024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-cjg6sk\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ed60\u003e: \n failed to create HCP cluster cluster-zstream-4-21-cjg6sk: failed waiting for cluster=\"cluster-zstream-4-21-cjg6sk\" in resourcegroup=\"rg-zstream-upgrade-4-21-h8qwfrzsn2zp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"name\": \"e4023d65-ddb6-4faf-abf2-0efb01d18c22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:38.211424873Z\",\n \"endTime\": \"2026-09-18T19:29:40.287386024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75713d57b69945b925dcf8e93e672f871273df96518a658f44ae78319e9b2052","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"3a36b2cb0e45db6b1b5adf537bf6a65a0dc2daa7ebc74fcf104789a0636dc715","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107a4a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nsq6nncs9jx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"name\": \"d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:51.185500713Z\",\n \"endTime\": \"2026-09-18T20:25:57.031163299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107a4a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nsq6nncs9jx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"name\": \"d18659c0-ccd6-4533-914c-00e40e166e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:51.185500713Z\",\n \"endTime\": \"2026-09-18T20:25:57.031163299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"761d0fa8553ef7bd44b1a22217c6da419a24c3dee00e22f2ceb7581c1d766865","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2c5047fbc800377676e3159b01faa4b14e97abaaef72bb1d0f7c177a6a8e963a","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ca440\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hx4h2xzxb7kg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"name\": \"e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:09.060511648Z\",\n \"endTime\": \"2026-09-18T20:24:17.026518831Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ca440\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hx4h2xzxb7kg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"name\": \"e5c499c9-a7e3-4879-bf2c-b0c897b22258\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:09.060511648Z\",\n \"endTime\": \"2026-09-18T20:24:17.026518831Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bf88482afa590949a19c3f9fff5eb3667ed0d7458569a1b3ce7d0458a14ab51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ac027a2fae7f9c119c5d13e433aca363beeb537c7f13236bce5b4509168d72e7","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7c320\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ntmzhdz9dn8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"name\": \"304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:03.328284727Z\",\n \"endTime\": \"2026-09-18T20:26:07.032768851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7c320\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ntmzhdz9dn8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"name\": \"304a6a3a-9f5b-4eef-9550-d0ba73b6ef78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:57:03.328284727Z\",\n \"endTime\": \"2026-09-18T20:26:07.032768851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80a4ddf1a1a99f8ee68dfb28a5326f3b971ff20de102e0de13972ae881f86611","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"8f8a4a3e3e075b660c1fb0534694eabcbec0ee97ddf1c7de50ae027a7c989e5e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e42a80\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-x6ksv4qd5hh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\",\n \"name\": \"0d11158a-c610-4ba0-b070-6f011685df27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.656789631Z\",\n \"endTime\": \"2026-09-18T18:35:23.922596176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e42a80\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-x6ksv4qd5hh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0d11158a-c610-4ba0-b070-6f011685df27\",\n \"name\": \"0d11158a-c610-4ba0-b070-6f011685df27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.656789631Z\",\n \"endTime\": \"2026-09-18T18:35:23.922596176Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80ab17b227e56f09e06a80ab71422ddb33e0236760a76a4c1b70f1688e33f9b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"55f11975675616706103b6b7daff6a4b423e0747ce59c5e7da66f3dc349981e7","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3e4e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5l7s7j7vcxkh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"name\": \"6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:09.380870167Z\",\n \"endTime\": \"2026-09-18T19:31:10.29248977Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3e4e0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5l7s7j7vcxkh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"name\": \"6488d30f-ab0b-48ea-a0ec-7c581c0e717f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:02:09.380870167Z\",\n \"endTime\": \"2026-09-18T19:31:10.29248977Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"81c7d6e579d2227b7a4c6ea47e6755cf7335f6d46baea2ddaa010b066e9a2744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ec2f8559597c2e2e96353159ad3c18cb0e7f4c1ae2d3f60c226c0178a7ef5d2b","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab4bc0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9x5xbv4nzw8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"name\": \"33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:03.940822361Z\",\n \"endTime\": \"2026-09-18T20:24:07.0250673Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab4bc0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9x5xbv4nzw8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"name\": \"33146f6d-4733-4ec4-b15d-3e685b83f09f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:03.940822361Z\",\n \"endTime\": \"2026-09-18T20:24:07.0250673Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84ac497f7e7f99cb1175249dc1d48e5993ac88dda3cbe590d6a110469fcb8764","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2bd526e04864e297297108f31c386e629877109b67a0cd5577f66e966f4820a0","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90f60\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-z57bbtc664tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\",\n \"name\": \"5f403839-210b-4649-acab-f3cebf6c199e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:56.042833825Z\",\n \"endTime\": \"2026-09-18T19:29:00.285060799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90f60\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-z57bbtc664tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5f403839-210b-4649-acab-f3cebf6c199e\",\n \"name\": \"5f403839-210b-4649-acab-f3cebf6c199e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:56.042833825Z\",\n \"endTime\": \"2026-09-18T19:29:00.285060799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"89ad59949d4cfe54b57f74b4db11d3f769ddaa5807f73b34f6c87be8fb46edbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"18cde222ab56ffb759e053d76ee6e19c739c516c2db2d311138d345c27569034","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e17a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qcdwg6ktcch6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"name\": \"b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:24.536461143Z\",\n \"endTime\": \"2026-09-18T18:33:33.916870368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e17a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qcdwg6ktcch6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"name\": \"b8db3dd6-3acf-4a8a-8882-bacb6f6fb9d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:24.536461143Z\",\n \"endTime\": \"2026-09-18T18:33:33.916870368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c00e4952c81686effa8b71a69a711a09892789e24e73cc03601bd75fafd09e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"8f472cc83c135324ad24ab195d238f2ea80f11a76ff49c1665f921d0f019a309","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"300283533309312abfaf8eb06c5bc243249dba952ac99795631a8cca96c1b04d","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9ab80\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qp94zrf9dxjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"name\": \"1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:31.378329777Z\",\n \"endTime\": \"2026-09-18T20:25:37.031824413Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9ab80\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qp94zrf9dxjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"name\": \"1869d89d-6245-485d-8118-d3fc6e9af9cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:31.378329777Z\",\n \"endTime\": \"2026-09-18T20:25:37.031824413Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"97bdc90c8a1291bc689a1b501c7b566934f460c4a1691ce224e4affe9cacea64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"eabc0e9f060f2e7a3e41ea442762aca89d3441bef1655db8dca9669defa6ff8c","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc6480\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p4nctxb27lkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\",\n \"name\": \"47247d9a-d966-4134-b361-76bb41af047f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.12537559Z\",\n \"endTime\": \"2026-09-18T18:33:23.915581975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc6480\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p4nctxb27lkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47247d9a-d966-4134-b361-76bb41af047f\",\n \"name\": \"47247d9a-d966-4134-b361-76bb41af047f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.12537559Z\",\n \"endTime\": \"2026-09-18T18:33:23.915581975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9904101b08db6937d4497fde1a011de06201d82d8d83d070b527ab492f051cfe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"980439aafe10d6730454288b03e5c4b325ad4d9a23209ccce9ea0925ca8727b5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-njqww7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af10c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-njqww7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-njqww7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jpqnx2lrkvjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"name\": \"b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:31.270212929Z\",\n \"endTime\": \"2026-09-18T19:30:40.29151616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-njqww7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af10c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-njqww7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-njqww7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jpqnx2lrkvjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"name\": \"b4fccd14-460f-43de-9e90-738bd6033ce0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:31.270212929Z\",\n \"endTime\": \"2026-09-18T19:30:40.29151616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b629ae8802b9eee0233f7dfc71c223fef82765ee22d82d8817a0a40fde6f397","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"aba4ff8b5edea60411faee257f0fcca1514022f58c987167a980974659ac599f","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b0380\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8dhk5572s4k2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"name\": \"0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:07:16.233683149Z\",\n \"endTime\": \"2026-09-18T18:36:20.211533932Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b0380\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8dhk5572s4k2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"name\": \"0b13dfc6-e74b-46ab-86f5-cfed93dba41f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:07:16.233683149Z\",\n \"endTime\": \"2026-09-18T18:36:20.211533932Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b6d1e1e893a32cbbdaf40f8ebcdec1d69bd66b92f9cc8bac39a913c0b69bb22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2a4e0071788107d1b0f28cafcae777b74d59c2ed92ca22e3caeb7564b1685be1","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14ec0\u003e: \n failed to create HCP cluster np-dg-minor-xwhdp5: failed waiting for cluster=\"np-dg-minor-xwhdp5\" in resourcegroup=\"rg-np-dg-minor-xwhdp5-592m9fw6ltl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"name\": \"35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:39.90066671Z\",\n \"endTime\": \"2026-09-18T20:25:47.031092558Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b14ec0\u003e: \n failed to create HCP cluster np-dg-minor-xwhdp5: failed waiting for cluster=\"np-dg-minor-xwhdp5\" in resourcegroup=\"rg-np-dg-minor-xwhdp5-592m9fw6ltl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"name\": \"35a3b7f4-5677-4ee3-8047-d4e91fdf91e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:56:39.90066671Z\",\n \"endTime\": \"2026-09-18T20:25:47.031092558Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a0f076a5c11daca409b4a8602d8e4ce9f653ba31dad49cecc3563150f920d98f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"9da2d69669521624ba370e29cdccd478000fd77fa5acf531ed0100da09e598f1","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010da5a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-6ppzp5frfmgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"name\": \"18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:25.031159118Z\",\n \"endTime\": \"2026-09-18T18:33:33.919612215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010da5a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-6ppzp5frfmgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"name\": \"18010207-11d3-4bad-8b26-bfe3951f9a05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:25.031159118Z\",\n \"endTime\": \"2026-09-18T18:33:33.919612215Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6575061b5d12438545743324e8367c9fea437febcee658a82868bc41212d5b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"12c88ab455c66941eb056387e08ab482c4399c448c1844b1db1d64ca2a21923b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-7ggpqb\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-7ggpqb: failed waiting for cluster=\"cluster-zstream-4-22-7ggpqb\" in resourcegroup=\"rg-zstream-upgrade-4-22-nfmkkc97chvl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"name\": \"14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.092484844Z\",\n \"endTime\": \"2026-09-18T19:29:40.290112371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-7ggpqb\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-7ggpqb: failed waiting for cluster=\"cluster-zstream-4-22-7ggpqb\" in resourcegroup=\"rg-zstream-upgrade-4-22-nfmkkc97chvl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"name\": \"14449ce9-b1ea-49dc-820c-31ab5444c62f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.092484844Z\",\n \"endTime\": \"2026-09-18T19:29:40.290112371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a82052cd828fbfd99b59f191e641dc7ad54d047f6e611710007c17f8f2b6e9f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"99a0fc3be3299d125a1312f83223671d90940dffaa62fee2e5b18166cfca67f4","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011faaa0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-smxqhhn278sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"name\": \"ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.39543422Z\",\n \"endTime\": \"2026-09-18T18:33:23.917604609Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011faaa0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-smxqhhn278sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"name\": \"ecd24d8c-9b96-49ef-9ee9-932851b8cc59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:22.39543422Z\",\n \"endTime\": \"2026-09-18T18:33:23.917604609Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9350470812b02e997b9125bc024f29804cb288e2de4809951bb8e962e38780f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"c9577e8c1b375d41d27d0e18df18642cc9e2a302455c27112af6cdf04b27da87","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-kg5zmv\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfa6c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-kg5zmv: failed waiting for cluster=\"cp-ystream-upgrade-4-22-kg5zmv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gsppncf9n78j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"name\": \"6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:34.659094453Z\",\n \"endTime\": \"2026-09-18T19:30:40.290599044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-kg5zmv\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfa6c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-kg5zmv: failed waiting for cluster=\"cp-ystream-upgrade-4-22-kg5zmv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gsppncf9n78j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"name\": \"6811d4f8-987c-40e9-a8fb-87314864cef2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:01:34.659094453Z\",\n \"endTime\": \"2026-09-18T19:30:40.290599044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b96757dbe2b79bd77de7e95dbc6b4fea759516a80e61fa6e5ab2ddd4a2b3e109","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"596b0ffa1cf24f21e4c89da0dd1bf8032daa39c9dfe890cbc997391f5e16d758","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dcf3c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-2v27j8w6flzw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"name\": \"3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:31.190318654Z\",\n \"endTime\": \"2026-09-18T18:33:33.918230691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dcf3c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-2v27j8w6flzw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"name\": \"3a82c0c6-4a42-4a86-a966-622844d30b20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:31.190318654Z\",\n \"endTime\": \"2026-09-18T18:33:33.918230691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cab1f907aa7e25850325ac850e1a2c316e4a3b3bbb5412886085dfe492af969f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"d9b87f83858caf3b228ef9b132bcd625c28601b6944272225765d7e4212bfbd4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-jspqshnm6b8s/cluster-candidate-4-20-26p7kj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d98400\u003e: \n failed to create HCP cluster cluster-candidate-4-20-26p7kj: failed waiting for cluster=\"cluster-candidate-4-20-26p7kj\" in resourcegroup=\"rg-candidate-4-20-jspqshnm6b8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"name\": \"670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.619385936Z\",\n \"endTime\": \"2026-09-18T19:29:20.288515017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-jspqshnm6b8s/cluster-candidate-4-20-26p7kj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d98400\u003e: \n failed to create HCP cluster cluster-candidate-4-20-26p7kj: failed waiting for cluster=\"cluster-candidate-4-20-26p7kj\" in resourcegroup=\"rg-candidate-4-20-jspqshnm6b8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"name\": \"670cbc64-10c9-40e8-ab17-ba6edf9a8da6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:14.619385936Z\",\n \"endTime\": \"2026-09-18T19:29:20.288515017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd20643ec408ce450bac1a54eba04042b26f3d3b640fbdbe5714cf8d7cec9897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"fcaef2f74196ab4f30c8d92bee5653f6add69a944685179ec76976f1aebee54b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6d540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rc84t6g5h54p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"name\": \"33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.507928047Z\",\n \"endTime\": \"2026-09-18T18:33:33.91760498Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6d540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rc84t6g5h54p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"name\": \"33766b12-eb33-46f0-b3b0-eea3719fedb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.507928047Z\",\n \"endTime\": \"2026-09-18T18:33:33.91760498Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd2c2a65757d4315af35d186835f3bcfa5bf77df9289f64d8aca90b0c3fcfb80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"cf464e2390b41d88251c211ae76f31739afd822d6d94414c2b9fbfc2f709feaa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-pc4nmpfmfd8n/cluster-candidate-5-0-8k9kgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008977c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-8k9kgf: failed waiting for cluster=\"cluster-candidate-5-0-8k9kgf\" in resourcegroup=\"rg-candidate-5-0-pc4nmpfmfd8n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\",\n \"name\": \"b151a448-87ca-4125-a955-ff96a58ce701\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:04.863718426Z\",\n \"endTime\": \"2026-09-18T19:29:10.287329661Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-pc4nmpfmfd8n/cluster-candidate-5-0-8k9kgf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008977c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-8k9kgf: failed waiting for cluster=\"cluster-candidate-5-0-8k9kgf\" in resourcegroup=\"rg-candidate-5-0-pc4nmpfmfd8n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b151a448-87ca-4125-a955-ff96a58ce701\",\n \"name\": \"b151a448-87ca-4125-a955-ff96a58ce701\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:04.863718426Z\",\n \"endTime\": \"2026-09-18T19:29:10.287329661Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cdde3d4c4687481087e9c788eb5069c6bdb0503fa226dba008b4f89e89f93ff6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"f52d9598c009df2683fa8e80fa13ffac13c236506f242fa3de8742e288cc9518","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-tpqdn5 at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e760\u003e: \n failed to create HCP cluster np-skip-minor-tpqdn5: failed waiting for cluster=\"np-skip-minor-tpqdn5\" in resourcegroup=\"rg-np-skip-minor-tpqdn5-tc97l4b65b5k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"name\": \"06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:52.428957278Z\",\n \"endTime\": \"2026-09-18T20:24:57.029688909Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-tpqdn5 at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e760\u003e: \n failed to create HCP cluster np-skip-minor-tpqdn5: failed waiting for cluster=\"np-skip-minor-tpqdn5\" in resourcegroup=\"rg-np-skip-minor-tpqdn5-tc97l4b65b5k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"name\": \"06ddea4f-de67-40c5-a480-9d00159e56c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:52.428957278Z\",\n \"endTime\": \"2026-09-18T20:24:57.029688909Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ced8f4053252ac14c6f638a3a6f16d54244333fd644369ee54b48a86cdb7cc82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"612c1b0bfa8c63e918534f55b46917bda5292bb668f43f5109397135c5f16839","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wlmbkw\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a24ce0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wlmbkw: failed waiting for cluster=\"cluster-zstream-5-0-wlmbkw\" in resourcegroup=\"rg-zstream-upgrade-5-0-88f465g4v8qg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"name\": \"8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.645326719Z\",\n \"endTime\": \"2026-09-18T19:29:40.288068036Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-wlmbkw\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a24ce0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-wlmbkw: failed waiting for cluster=\"cluster-zstream-5-0-wlmbkw\" in resourcegroup=\"rg-zstream-upgrade-5-0-88f465g4v8qg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"name\": \"8399e46e-e9e9-480e-b60a-122ad9e52531\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:00:37.645326719Z\",\n \"endTime\": \"2026-09-18T19:29:40.288068036Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d40139f7ef37bcb47627e083dba680a368db24cbb9aa90f6514b6709ddaa3f6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"8e3503c7dac56f3fd400f32d8b3f5b71ecf5acd0c55bc2d5e52b0b130348d093","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-phwsxm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dc9c0\u003e: \n failed to create HCP cluster np-noedge-phwsxm: failed waiting for cluster=\"np-noedge-phwsxm\" in resourcegroup=\"rg-np-noedge-phwsxm-4bv8jxxfz58h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"name\": \"ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:33.085250323Z\",\n \"endTime\": \"2026-09-18T20:24:37.028572057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-phwsxm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dc9c0\u003e: \n failed to create HCP cluster np-noedge-phwsxm: failed waiting for cluster=\"np-noedge-phwsxm\" in resourcegroup=\"rg-np-noedge-phwsxm-4bv8jxxfz58h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"name\": \"ac9b5fdb-7eb8-486a-9321-c2c553703d36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:33.085250323Z\",\n \"endTime\": \"2026-09-18T20:24:37.028572057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e317527bcfba5b41113f4d3f3df8dfb232d83b2119b8223f2b165223b4571a5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"b09c9c83b49154876ea9edaee019cac3ced0de0e27ddbe6889ee50f6b423f2ad","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e260\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ld4s6tgxr5vg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"name\": \"a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:16:16.855336614Z\",\n \"endTime\": \"2026-09-18T19:45:20.354805698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e260\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ld4s6tgxr5vg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"name\": \"a000d2ab-54f0-4334-b41d-9e9451140d91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:16:16.855336614Z\",\n \"endTime\": \"2026-09-18T19:45:20.354805698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ead2169eefc3a2c8aec7cc6799696a576e5414661e3b0a307a5b0b669d06504c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"55df98f2e805e8d535825529f230ec77e5caf64b117f366392092a2ef0f04238","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c425a0\u003e: \n failed to create HCP cluster np-downgrade-7d26bl: failed waiting for cluster=\"np-downgrade-7d26bl\" in resourcegroup=\"rg-np-downgrade-7d26bl-7qmvc9mdqdmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"name\": \"02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:45.411002197Z\",\n \"endTime\": \"2026-09-18T20:24:47.028298292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c425a0\u003e: \n failed to create HCP cluster np-downgrade-7d26bl: failed waiting for cluster=\"np-downgrade-7d26bl\" in resourcegroup=\"rg-np-downgrade-7d26bl-7qmvc9mdqdmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"name\": \"02ee3c59-eff4-48f9-8e78-44b142853e94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:45.411002197Z\",\n \"endTime\": \"2026-09-18T20:24:47.028298292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec56c0b165a44db2deaa54ac9c25317f315fa48b0d4c8a782cfd702c07e6fe87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"c71f8c34107674e9b83007fd4c4382acbf933644d3d70e04b10c2bce3d213437","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011ed1a0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"name\": \"d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:57.828166643Z\",\n \"endTime\": \"2026-09-18T18:35:03.924193327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011ed1a0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"name\": \"d4795f3f-29d6-4f65-a818-0470837029c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:05:57.828166643Z\",\n \"endTime\": \"2026-09-18T18:35:03.924193327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f0a818c35c7b74c13a9a05f8d18ada59065ca609029af910ebce486e547b9549","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"9253c3493335b5dfd398622bbe3451ad8bddf74417d1a8c0037aece178838cde","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-59kkxj with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90940\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-59kkxj: failed waiting for cluster=\"np-version-upgrade-cluster-59kkxj\" in resourcegroup=\"rg-np-version-upgrade-59kkxj-strjcq5cf28t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"name\": \"c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:19.582015951Z\",\n \"endTime\": \"2026-09-18T20:24:27.027192644Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-59kkxj with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90940\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-59kkxj: failed waiting for cluster=\"np-version-upgrade-cluster-59kkxj\" in resourcegroup=\"rg-np-version-upgrade-59kkxj-strjcq5cf28t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"name\": \"c6cb01a7-ddc6-483a-83c4-b3f161303a6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:19.582015951Z\",\n \"endTime\": \"2026-09-18T20:24:27.027192644Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f403f49fafa7eaba741dba44bdfbac7a3fcb424d950c14ac38fc7bc11208982c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"19596332cc8edb7c2fe7ff05b62421eeeea53668cb594d6f172e695930cfc93e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016acc40\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t8z82lzs48sr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"name\": \"ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.431481456Z\",\n \"endTime\": \"2026-09-18T18:33:33.917230374Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016acc40\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t8z82lzs48sr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"name\": \"ee4e2ffd-758b-447c-95c9-acf961276e55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:04:32.431481456Z\",\n \"endTime\": \"2026-09-18T18:33:33.917230374Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f457631bf2de98a8eb53777c8928d27f0e7c666d176e4ab60a495aafbeb039f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"2e5039616cb1c8d7f4ac14c50b2a06f1fe40081131c5acd543f28d2cc514abec","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015aed60\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vxq45g66n2fb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"name\": \"1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:43.251936528Z\",\n \"endTime\": \"2026-09-18T19:28:50.284453068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015aed60\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vxq45g66n2fb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"name\": \"1aa4f569-8351-4ab6-8cbb-fa86fa721f61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:59:43.251936528Z\",\n \"endTime\": \"2026-09-18T19:28:50.284453068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f721de984f1e9d2adc21d569363335cf38d2031abf0e310bc9cec2d464a4c943","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","failed_at":"e2e","signature_id":"ccebff601285f1ea3ac04ae7a355282a8c0e14e4f8eb582eebc840c11f6d9a08","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gs8fc2 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbc920\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gs8fc2: failed waiting for cluster=\"np-version-upgrade-cluster-gs8fc2\" in resourcegroup=\"rg-np-version-upgrade-gs8fc2-flflx6nnvpbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\",\n \"name\": \"602c1354-35fa-4664-9d86-220f8f644a46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:14.360447552Z\",\n \"endTime\": \"2026-09-18T20:24:17.027918855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gs8fc2 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbc920\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gs8fc2: failed waiting for cluster=\"np-version-upgrade-cluster-gs8fc2\" in resourcegroup=\"rg-np-version-upgrade-gs8fc2-flflx6nnvpbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/602c1354-35fa-4664-9d86-220f8f644a46\",\n \"name\": \"602c1354-35fa-4664-9d86-220f8f644a46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:14.360447552Z\",\n \"endTime\": \"2026-09-18T20:24:17.027918855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":54,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7"],"matched_occurrences":54,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100939709793439744","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T13:27:14Z","started_at":"2026-09-18T13:27:14Z","completed_at":"2026-09-18T16:03:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T16:11:55.679083178Z","timing_metadata_checked_at":"2026-09-18T16:11:55.679083178Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T16:11:55.683053992Z","region_metadata_checked_at":"2026-09-18T16:11:55.683053992Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100939566205636608","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T13:26:34Z","started_at":"2026-09-18T13:26:34Z","completed_at":"2026-09-18T15:54:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T16:01:33.186908727Z","timing_metadata_checked_at":"2026-09-18T16:01:33.186908727Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T16:01:33.19001214Z","region_metadata_checked_at":"2026-09-18T16:01:33.19001214Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100939394377584640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T13:25:59Z","started_at":"2026-09-18T13:25:59Z","completed_at":"2026-09-18T15:57:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T16:01:33.059102212Z","timing_metadata_checked_at":"2026-09-18T16:01:33.059102212Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T16:01:33.065093236Z","region_metadata_checked_at":"2026-09-18T16:01:33.065093236Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:26:04Z","started_at":"2026-09-18T10:26:04Z","completed_at":"2026-09-18T12:54:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.688508858Z","timing_metadata_checked_at":"2026-09-18T15:42:48.688508858Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.691603169Z","region_metadata_checked_at":"2026-09-18T15:42:48.691603169Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7a16b7d73638182fb5e8d36595d4f82bd635e5aa75a7038b4e6eb77adc46261e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","occurred_at":"2026-09-18T10:26:04Z","failed_at":"e2e","signature_id":"3269dcb6be86d6a6047ecae085ca193c45aec85e3774519113b1593c115dfa87","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a45890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a45890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:20:35Z","started_at":"2026-09-18T10:20:35Z","completed_at":"2026-09-18T12:47:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.548336754Z","timing_metadata_checked_at":"2026-09-18T15:42:48.548336754Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.551590466Z","region_metadata_checked_at":"2026-09-18T15:42:48.551590466Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0fb792a233958dd2d09e4c8ba1ccd3cc8b8e162437215b1cf4005751b59010f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","occurred_at":"2026-09-18T10:20:35Z","failed_at":"e2e","signature_id":"d7f2aa1ec90b6da0508d97952c26f5e68a44fec6becea93e11a0c8acd2ecf80a","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100891307781132288","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-18T10:14:55Z","started_at":"2026-09-18T10:14:55Z","completed_at":"2026-09-18T12:50:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.421816799Z","timing_metadata_checked_at":"2026-09-18T15:42:48.421816799Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.431526734Z","region_metadata_checked_at":"2026-09-18T15:42:48.431526734Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T10:10:35Z","started_at":"2026-09-18T10:10:35Z","completed_at":"2026-09-18T12:36:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.204624418Z","timing_metadata_checked_at":"2026-09-18T15:42:48.204624418Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.20782373Z","region_metadata_checked_at":"2026-09-18T15:42:48.20782373Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c257c9cf8f6ee1561328a5d8652a6d4630c8c215eb8d7deb94b19d07bff1dbfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","occurred_at":"2026-09-18T10:10:35Z","failed_at":"e2e","signature_id":"5533b31e4276757a4afc74d1ad67148e45245ccef8ea260d65b9e08077693a1d","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088e570\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088e570\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T05:58:50Z","started_at":"2026-09-18T05:58:50Z","completed_at":"2026-09-18T08:28:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.992752057Z","timing_metadata_checked_at":"2026-09-18T15:42:47.992752057Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:47.995942368Z","region_metadata_checked_at":"2026-09-18T15:42:47.995942368Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cd86f0ae9a6585720a683c4ec931484238a8d28bf0831d63a340257ec5268052","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","occurred_at":"2026-09-18T05:58:50Z","failed_at":"e2e","signature_id":"7cb7501ef4b819fdc17f5b5e3ca2619d486971f5a3c25a5533b8ff273a412172","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T02:00:17Z","started_at":"2026-09-18T02:00:17Z","completed_at":"2026-09-18T04:19:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:50.478297492Z","timing_metadata_checked_at":"2026-09-18T15:42:50.478297492Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:50.482649407Z","region_metadata_checked_at":"2026-09-18T15:42:50.482649407Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"442af769c0c7e3c0a8fb59afc4f8ad6ca5129510396948fa20c1e16b0ee3f399","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"6e4745afece375c35c57231f904892210b15b74988a6f2baceda810fa43da8e5","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001092648\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001092648\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4436f72f86dcdf7d5b39b762f07fd973095e73e991a63abf4213f7c8428ef204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"60cc795ca9532d84d0a6a8c864d8b05a2f5285c1bfd6c8aa612a88d8890d8bb5","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"953a178f8aa729db097bdc70aca88aeda09ff4fcc8e7558bccc494ea7e775e80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"6fd32498c414e944f0c010bd303fd297e25f438aa938164d7b89331e4fe1bf3e","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012e0990\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012e0990\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-sxsc7bxhvj2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}},{"row_id":"d3810f538bd4859e129bbd454ece2c7089a38c3720080d94e45f571849456219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","failed_at":"e2e","signature_id":"8be7c23a1e56f277787ec21fa08eb46e46af4b8580d42385a8b71372839091d6","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0cd20\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-hg2tcz\" in resourcegroup=\"rg-np-dg-minor-hg2tcz-n7pnlmn2w4pv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"name\": \"bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:32:13.01856929Z\",\n \"endTime\": \"2026-09-18T03:51:16.292258995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0cd20\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-hg2tcz\" in resourcegroup=\"rg-np-dg-minor-hg2tcz-n7pnlmn2w4pv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"name\": \"bd7e5bf4-841e-4289-9a4b-a34d164ba80f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:32:13.01856929Z\",\n \"endTime\": \"2026-09-18T03:51:16.292258995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-18T01:44:31Z","started_at":"2026-09-18T01:44:31Z","completed_at":"2026-09-18T04:25:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.858222373Z","timing_metadata_checked_at":"2026-09-18T15:42:47.858222373Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:47.861839386Z","region_metadata_checked_at":"2026-09-18T15:42:47.861839386Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"15361385a7407ab14f0a513e66c95ec47f5e75567c7e48cc4eaa99de087ec84f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","failed_at":"e2e","signature_id":"736f2b0814fabfc59b7e462d9755a0e748e4d396c2743f81540491fd582dab16","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"39025eae0aa723d1be917bb4455954c57ec535d64f978439377e18b5ca3efe61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","failed_at":"e2e","signature_id":"f164ce20210c5901433b68d8b32c7bafd9db9fbdb4b9f088ddfe4a53eb8ff36e","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001176ff0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001176ff0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cxcwb4twxbz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}},{"row_id":"ee0ca8706b64a81e3310ae86225db08f16064fdc710fab225376bf191551ae61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","failed_at":"e2e","signature_id":"812ff7352462ae5b2ffc2a98177a772e72a7a199983bda3af2609cfd08e8cb77","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001005b20\u003e: \n failed to create NodePool np-pd9r72: failed waiting for nodepool=\"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\" in resourcegroup=\"rg-candidate-4-20-7nvxhnmpb6qr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"name\": \"e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:55:35.193527856Z\",\n \"endTime\": \"2026-09-18T03:14:42.13127378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001005b20\u003e: \n failed to create NodePool np-pd9r72: failed waiting for nodepool=\"np-pd9r72\" for cluster \"cluster-candidate-4-20-pd9r72\" in resourcegroup=\"rg-candidate-4-20-7nvxhnmpb6qr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"name\": \"e62533b3-e87f-4306-b7fa-7e8f2f79508a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:55:35.193527856Z\",\n \"endTime\": \"2026-09-18T03:14:42.13127378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}