{"meta":{"date":"2026-10-17","timezone":"UTC","generated_at":"2026-10-29T05:06:31Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-17T23:00:12Z","started_at":"2026-10-17T23:00:12Z","completed_at":"2026-10-18T00:40:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-18T00:55:02.093932507Z","timing_metadata_checked_at":"2026-10-18T00:55:02.093932507Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-18T00:55:02.097676495Z","region_metadata_checked_at":"2026-10-18T00:55:02.097676495Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"0241e10f36043812e22c6cb7058f06a77caf43bc22db02d70d119b6ee87f194c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"c83d4a9782166f7b71d9629819e951ac19a26856daa1b37932ca035306b029f8","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc36df70ac60\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tx2g82pvpq9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tx2g82pvpq9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 1baca0c7-bdc6-4a30-871e-d4caf102be39\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc36df70ac60\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tx2g82pvpq9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tx2g82pvpq9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 1baca0c7-bdc6-4a30-871e-d4caf102be39\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4611c9a53108703cdbbc3fe44037d929d2081a6d81f69adf0b19bc0afcccb897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"746e6c02db8bad8c95708c9d534e32851cd04c4fa3111297a27d44f9b56cec47","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:509]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1b882be0fa60\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:509]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1b882be0fa60\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"652bb742e0e0042d559ce87ce100cf3c4162742b9cd17fe55dc0be2147bc586c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"6b313ad3d4a4a54709019be4e70c8735a875468935b00a4efc9bda0302c0bba4","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1968c8f0e7e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bxghpr2sckgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bxghpr2sckgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: b29c363b-dd94-455c-888b-9009557b53bb\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1968c8f0e7e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bxghpr2sckgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bxghpr2sckgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: b29c363b-dd94-455c-888b-9009557b53bb\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9defd043b614040b4450c3d299dca321ecc0f64a8d27f3d5edd546639193e34c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"888af4796757ff564191f1aed7ec1b7770c322ff5a7b10374b74ecde0cb4472b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x368f4638cd20\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-jdskq89krgsv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-jdskq89krgsv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 48ea241b-c8cc-4055-8ba1-09602a9c1346\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x368f4638cd20\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-jdskq89krgsv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-jdskq89krgsv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 48ea241b-c8cc-4055-8ba1-09602a9c1346\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ac56b0de5cfc09487e7e9fbcc59b3f4c74b1295b0f832343dc8681fc54deee75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"bd5409f40fb9b8e6a263a76353a71c154a70a62d9bec8cae16a2bea040217ef6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x7c66573fa40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hjlvlzn5gdv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hjlvlzn5gdv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 96b59387-b750-4f64-b0f1-23237235b1b2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x7c66573fa40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hjlvlzn5gdv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hjlvlzn5gdv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 96b59387-b750-4f64-b0f1-23237235b1b2\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c3eaad4ae77a43166bb5dd64e82720ade2a1b2adc16845a845302ea036cdf9f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"0fc96e29b1775849ae0486010f066562b18274f101f007bfa1bcece1bcd2efb4","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x10d3b5409cb0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-k7dtdk5fgs6g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-k7dtdk5fgs6g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 0c284d71-dd2e-4473-8bbf-bb91bb8092f4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x10d3b5409cb0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-k7dtdk5fgs6g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-k7dtdk5fgs6g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 0c284d71-dd2e-4473-8bbf-bb91bb8092f4\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c762760c205f47e6493cbd717217079b4607b85272215978c3630bebb9ce490a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"427c7c67fa1ece4b39b523cda25fe446af89c2389a2bf614a43c48bbbf0ed716","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x122fed28cd60\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.004s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.004s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x122fed28cd20\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x122fed28cce0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x122fed28cd60\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.004s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.004s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x122fed28cd20\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x122fed28cce0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40fd7a6950bf8f4a3acf4f0624934387f01a457fbf420dfeb26442bd2e3882ba","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"dd290cf39321f60a523c98f4b931696e6000d54d398c8449d69ef5416704b335","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"9e2b5c623056f04c4b824e6f6e84658b1ebd526fd88144d3c583c0a9e2b7be96","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x26a811ac2138\u003e: \n    resource group \"rg-nightly-4-20-t74cxvx2rdnz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-cb8gb2\" in resourcegroup=\"rg-nightly-4-20-t74cxvx2rdnz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51202163-2b8d-4c4b-abd9-e60c454750ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51202163-2b8d-4c4b-abd9-e60c454750ef\",\n      \"name\": \"51202163-2b8d-4c4b-abd9-e60c454750ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-17T23:43:19.794851177Z\",\n      \"endTime\": \"2026-10-18T00:07:19.354701033Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51202163-2b8d-4c4b-abd9-e60c454750ef\n            correlation ID: d4f725b2-2286-488f-91cf-637285920d3b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n    \u003c*errors.joinError | 0x26a811ac2138\u003e: \n    resource group \"rg-nightly-4-20-t74cxvx2rdnz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-cb8gb2\" in resourcegroup=\"rg-nightly-4-20-t74cxvx2rdnz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51202163-2b8d-4c4b-abd9-e60c454750ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51202163-2b8d-4c4b-abd9-e60c454750ef\",\n      \"name\": \"51202163-2b8d-4c4b-abd9-e60c454750ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-17T23:43:19.794851177Z\",\n      \"endTime\": \"2026-10-18T00:07:19.354701033Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51202163-2b8d-4c4b-abd9-e60c454750ef\n            correlation ID: d4f725b2-2286-488f-91cf-637285920d3b\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ebbb69cb9ed47c033ebf34b38fd75c121d50dc5d28239fa92e2813d822718c77","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd7889624882dd9212a02f9951ffd69b4bec561f2b1f3ebc310fd06a58f8a73c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2111593145538973696","occurred_at":"2026-10-17T23:00:12Z","failed_at":"e2e","signature_id":"b0d3841d4a984de3c5ebae440c2e284dfd5fa10e426e984646f0504573da4d44","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:343]: Timed out after 300.003s.\nDegraded condition did not converge to False\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:323 with:\nexternal auth Properties.Status was nil\nExpected\n    \u003c*armredhatopenshifthcp.ResourceStatus | 0x0\u003e: nil\nnot to be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:343]: Timed out after 300.003s.\nDegraded condition did not converge to False\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:323 with:\nexternal auth Properties.Status was nil\nExpected\n    \u003c*armredhatopenshifthcp.ResourceStatus | 0x0\u003e: nil\nnot to be nil","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5a972289ad4ab45b4713665dee826b535d8a68c11c1b3e9a148a15bbd67840c0","failure_pattern":"external auth \u003cexternal-auth\u003e.Status was nil","search_query":"external auth Properties.Status was nil"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","40fd7a6950bf8f4a3acf4f0624934387f01a457fbf420dfeb26442bd2e3882ba","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5a972289ad4ab45b4713665dee826b535d8a68c11c1b3e9a148a15bbd67840c0","ebbb69cb9ed47c033ebf34b38fd75c121d50dc5d28239fa92e2813d822718c77"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2111276046056689664","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-17T02:00:09Z","started_at":"2026-10-17T02:00:09Z","completed_at":"2026-10-17T04:01:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-17T04:15:12.729027528Z","timing_metadata_checked_at":"2026-10-17T04:15:12.729027528Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-17T04:15:12.733228114Z","region_metadata_checked_at":"2026-10-17T04:15:12.733228114Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1d3ebd097671aeeda9e522fa059d3902f4543ad9266707ebf1094e46189af0ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2111276046056689664","occurred_at":"2026-10-17T02:00:09Z","failed_at":"e2e","signature_id":"912c71b494a8545a23fe4d5d3bd813a2703e883d8b2ca24f6c6ef2ee9f1f1e44","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:343]: Timed out after 300.001s.\nDegraded condition did not converge to False\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:323 with:\nexternal auth Properties.Status was nil\nExpected\n    \u003c*armredhatopenshifthcp.ResourceStatus | 0x0\u003e: nil\nnot to be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:343]: Timed out after 300.001s.\nDegraded condition did not converge to False\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:323 with:\nexternal auth Properties.Status was nil\nExpected\n    \u003c*armredhatopenshifthcp.ResourceStatus | 0x0\u003e: nil\nnot to be nil","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5a972289ad4ab45b4713665dee826b535d8a68c11c1b3e9a148a15bbd67840c0","failure_pattern":"external auth \u003cexternal-auth\u003e.Status was nil","search_query":"external auth Properties.Status was nil"}},{"row_id":"56e4c892ece21c2ba6d94ce6d5be2c13f977044eba0aa05d1a56f453b32b1971","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2111276046056689664","occurred_at":"2026-10-17T02:00:09Z","failed_at":"e2e","signature_id":"d7903f5717827e61c38008e092bba7c734058dec2338bb19fd227a91373d715b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:509]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2da77e7d1600\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:509]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2da77e7d1600\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"ad35d88575d8857f38b29bb22f1eef872e7075fdd0277b4e83198bd0a32a67c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2111276046056689664","occurred_at":"2026-10-17T02:00:09Z","failed_at":"e2e","signature_id":"bf541579b353cf5bc164488d30b73efb4d70839538df99e7360e24ef99d83c70","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3c7ac9b4ac00\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.01s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.01s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3c7ac9b4abc0\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x3c7ac9b4ab80\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3c7ac9b4ac00\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.01s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.01s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3c7ac9b4abc0\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x3c7ac9b4ab80\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40fd7a6950bf8f4a3acf4f0624934387f01a457fbf420dfeb26442bd2e3882ba","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","40fd7a6950bf8f4a3acf4f0624934387f01a457fbf420dfeb26442bd2e3882ba","5a972289ad4ab45b4713665dee826b535d8a68c11c1b3e9a148a15bbd67840c0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}