{"meta":{"date":"2026-10-03","timezone":"UTC","generated_at":"2026-10-30T23:05:14Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T23:00:45Z","started_at":"2026-10-03T23:00:45Z","completed_at":"2026-10-04T00:39:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T00:47:38.949027707Z","timing_metadata_checked_at":"2026-10-04T00:47:38.949027707Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T00:47:38.952123897Z","region_metadata_checked_at":"2026-10-04T00:47:38.952123897Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"19a372277c7dece463b21300f514a070e5bef1b43d45c7b41d79e08ac45a436e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"467ff9b3fff0fba450d4251bf1cedce098cd2ea5e773aae9c7dab251c9546c13","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-h26qrg\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0x34bdd7d932c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5fxfbtp77wdj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-h26qrg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-h26qrg\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0x34bdd7d932c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5fxfbtp77wdj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-h26qrg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2c9acf0a041e35db9a096a5e9a1c6e5fde6548c33d99abc709b51e8f0e31e3b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"eb95f17360545876d289dee37a778611d4344608288ffa82220daa2648023378","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x156bbec7cb0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-48mh6dg9zlr8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x156bbec7cb0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-48mh6dg9zlr8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2e5733bb82a9d59382f96eebdd4b8d060ddc95729d17fc114d2de16fdd18782d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"b1e65d263ab4ed8d6d367a1d2b0e22cf65bd386883e59e308c33eef10b3fdd3c","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x210c33ec90b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8fsbbrnx57fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x210c33ec90b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8fsbbrnx57fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"47c1e63ee81b5efdfd595d023a3110e0ef02a7c95f3c70194bafd2f049aebe23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"46e00fd4ba016237d54a5c172dba5d7ae1620c42bcb5d60c65e763e9df2f79cd","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xd97df9bad80\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cskpwn77dkwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xd97df9bad80\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cskpwn77dkwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"511207853c6006a6c2ead9c333f6e259cb9ed0a167e05a61432a0e5bf3ba16d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"6cf60cc34982ca354ed254e7db99f8c3c17c9ef039ff68ffc95b5e18a6f44b25","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x781abe80140\u003e: \n failed to create NodePool np-4kgdcw: failed starting nodepool creation \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\" in resourcegroup=\"rg-nightly-4-20-4t8l44cr8nbr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-4t8l44cr8nbr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-4kgdcw/nodePools/np-4kgdcw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x781abe80140\u003e: \n failed to create NodePool np-4kgdcw: failed starting nodepool creation \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\" in resourcegroup=\"rg-nightly-4-20-4t8l44cr8nbr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-4t8l44cr8nbr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-4kgdcw/nodePools/np-4kgdcw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8783ccf2993e5a87020d3f4664b66da2f07940cec670f0bb57155f301cf26316","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"df520e23539da9e8ef8e4034c818cfb36af7d49a348b14cd808580a60fa3775f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d7c17881b00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hn22fz5qv7wv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d7c17881b00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hn22fz5qv7wv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ac2b9a0207d941d433eab33b72bf782827e619c5ef3461ccd1badd6b9be82f4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"8128113059a95965bef3d3c445b7c0cac5fe7442cb1f7985e9426b72d3507e7d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n \u003c*errors.joinError | 0x29a4ab6ac030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n \u003c*errors.joinError | 0x29a4ab6ac030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized"}},{"row_id":"bf44aacc67413eb15a9fa079b1b4221883bf2f5cc4919481437a4e28a76d6108","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"8af9e2c89645a4bf3765a33eab4e694ab3d6205ee2dba4b2403cbbe04e4bb04b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d5efc20c00\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b4wh48\" in resourcegroup=\"rg-np-version-upgrade-b4wh48-bzk7j574q45w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b4wh48-bzk7j574q45w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-b4wh48/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d5efc20c00\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b4wh48\" in resourcegroup=\"rg-np-version-upgrade-b4wh48-bzk7j574q45w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b4wh48-bzk7j574q45w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-b4wh48/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c3f8e18fe302ce62947bf15e89418b8cd8f5154bb0f0602106f2189584012eae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"d23e0af5f66a9d21ebfe7f67bab5ec265f1980b1c4649c6850acb948a72ccc0c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-54csls\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n \u003c*exported.ResponseError | 0x2086109860c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-hx8x6fq8plbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-54csls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-54csls\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n \u003c*exported.ResponseError | 0x2086109860c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-hx8x6fq8plbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-54csls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c6b165759bfdeca8f65ae4807ff00138c6e9f7674c570efc571d070ad4707f7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"ab51303944522693c0b33104b7c92d86be3eb8c3466ad20546a9b77e031ae5b1","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cb7bb3ff960\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pf8r4pxpb5hl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-pf8r4pxpb5hl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cb7bb3ff960\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pf8r4pxpb5hl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-pf8r4pxpb5hl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"dc35506b62aed78f873b23aa0ab86637435f48eb9d5ff5bde724c455df7fde2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"103fa735d194341e5747569e803885dd3d43db0551359672451398c565ccf594","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xcf8828ab170\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-crv2w79vmlxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xcf8828ab170\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-crv2w79vmlxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"dfb2851a4812946146384b0bcd660048a5e5e45291bd7367319187e07771647e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"4cf3eda2914d3b1a9083e46adec66b3f55112482155fd5d64fc7f418ec60e0b0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x325c79dc0060\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-987nc9dd4vhq\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-987nc9dd4vhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x325c79dc0060\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-987nc9dd4vhq\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-987nc9dd4vhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310818076299264","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T09:10:07Z","started_at":"2026-10-03T09:10:07Z","completed_at":"2026-10-03T12:00:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:07:44.065642534Z","timing_metadata_checked_at":"2026-10-03T12:07:44.065642534Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:07:44.06861542Z","region_metadata_checked_at":"2026-10-03T12:07:44.06861542Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T09:09:36Z","started_at":"2026-10-03T09:09:36Z","completed_at":"2026-10-03T11:49:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:57:20.514770193Z","timing_metadata_checked_at":"2026-10-03T11:57:20.514770193Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:57:20.51866138Z","region_metadata_checked_at":"2026-10-03T11:57:20.51866138Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5c45c8cb2cb6d588bdcb890095f27ed5f60f4fa514ca17570582276db85ad95d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","occurred_at":"2026-10-03T09:09:36Z","failed_at":"e2e","signature_id":"3bc3377f053adf5e02635272297b8adacea15f1260ea4f1c4bf49a21872672c8","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2861f94db110\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-s8tjmf2mg47r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"name\": \"a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:45:52.709942431Z\",\n \"endTime\": \"2026-10-03T10:04:55.390706618Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-mf948-nwnch: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-mf948-nwnch: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2861f94db110\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-s8tjmf2mg47r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"name\": \"a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:45:52.709942431Z\",\n \"endTime\": \"2026-10-03T10:04:55.390706618Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-mf948-nwnch: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-mf948-nwnch: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"743311f7cf757f8f0a1a9074c2916984b50d656001d8540e786322767e33a3ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","occurred_at":"2026-10-03T09:09:36Z","failed_at":"e2e","signature_id":"13af940fe84aa9225f10173afd7fedb188865ae08fb26ab20d44731489099168","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e8cc763a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wrvrktrj9pn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"name\": \"a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:41:16.318835994Z\",\n \"endTime\": \"2026-10-03T10:00:22.618172935Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e8cc763a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wrvrktrj9pn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"name\": \"a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:41:16.318835994Z\",\n \"endTime\": \"2026-10-03T10:00:22.618172935Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106309368474505216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T09:04:21Z","started_at":"2026-10-03T09:04:21Z","completed_at":"2026-10-03T11:38:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:46:32.104250608Z","timing_metadata_checked_at":"2026-10-03T11:46:32.104250608Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:46:32.106568501Z","region_metadata_checked_at":"2026-10-03T11:46:32.106568501Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106302957979963392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T08:38:53Z","started_at":"2026-10-03T08:38:53Z","completed_at":"2026-10-03T11:21:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:25:37.799730289Z","timing_metadata_checked_at":"2026-10-03T11:25:37.799730289Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:25:37.802785579Z","region_metadata_checked_at":"2026-10-03T11:25:37.802785579Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c9b7b2ace7f478670b02aa26aba828b8e4f1c801082feb818e55bb81523fa333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106302957979963392","occurred_at":"2026-10-03T08:38:53Z","failed_at":"e2e","signature_id":"adee8926a72629f9d64b7a77f2665eb3670a4af4a352e102d836fb3757b3bc93","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a380b9ff820\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"name\": \"1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:22:50.222769152Z\",\n \"endTime\": \"2026-10-03T10:41:50.376552801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a380b9ff820\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"name\": \"1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:22:50.222769152Z\",\n \"endTime\": \"2026-10-03T10:41:50.376552801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:00:44Z","started_at":"2026-10-03T02:00:44Z","completed_at":"2026-10-03T04:05:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:14:38.273664393Z","timing_metadata_checked_at":"2026-10-03T04:14:38.273664393Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T04:14:38.277027182Z","region_metadata_checked_at":"2026-10-03T04:14:38.277027182Z"},"failed_at":["e2e"],"failed_test_count":22,"occurrences":[{"row_id":"0c9dcf6dab558084a2a9ef2ffb5928acaae8895b52a9a68a3083498e01c99d34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"7b4cf1cf1f356f8a0bd6bfb20a926fe794e222e24e9098fbe0eb9156b1ec6573","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v6zlk7\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x3aa50a408f30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-7fls5z997c4n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-v6zlk7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v6zlk7\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x3aa50a408f30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-7fls5z997c4n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-v6zlk7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"10bf897673aab557e061392b4e7b085cf06cf71d2e87436b087d4d916dcdf7a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"366a1991051b1a37601646ae4edff31e67686f7599119c046ea638bd894e910e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-msl8ps\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x21a7a658f5c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-l7wmbxqck8gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-msl8ps\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-msl8ps\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x21a7a658f5c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-l7wmbxqck8gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-msl8ps\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1575987683f490cd4bed1fc8f8ae7fbc66348f4139e4c35f3fe89965b1730630","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"f4f3787f7980560a701583c4b17b2aa434c0cb2c95496fa5661ef575b0ba94d2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9hrm6z\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1415c20a5500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-cck68gj6qfsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9hrm6z\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9hrm6z\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1415c20a5500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-cck68gj6qfsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9hrm6z\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"15bdae944280f7615b0ec41135ff01e6844852a38792e237a7195cecfaaa88db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"8c999722171344de70dda1803d5579aed1e0dbc4ecd2b2c5ea3d0e6b792cbf9c","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x61296253c20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-96mcz9ktg57h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x61296253c20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-96mcz9ktg57h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1773cbbaa075a165279663c958753492642f49599aee70aebc5d049d9b8b3d87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c6f98fc883c08bd20a81c2a2a9718e59825e4839ac864d806b4647a01ab008bd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-x4v9ss\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x5a0752a7c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-9j7fg4k8csln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-x4v9ss\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-x4v9ss\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x5a0752a7c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-9j7fg4k8csln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-x4v9ss\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"18fb1a91b8a224a94f4434293ef344187f48338df84d363c146ff6a9e4f45bce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"28086d43cfa72152275388034cc851903b47aa1ca5775b0f6fa80ca38a6ab4fb","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ed03c0ba1c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-nrr89q\" resourcegroup=\"rg-np-dg-minor-nrr89q-d5btp9lwtkwk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nrr89q-d5btp9lwtkwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nrr89q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ed03c0ba1c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-nrr89q\" resourcegroup=\"rg-np-dg-minor-nrr89q-d5btp9lwtkwk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nrr89q-d5btp9lwtkwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nrr89q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3f4aac6bf9d1dd700cf6ea0451b03f1030eff54fe6dd1fd9c78540e5dcd9dea2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"7346747ba389cbf83d28f49662dfd4c31ae1261a21f1a9894630a81cb52c0065","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-w78qcb\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x39bf51e7ba70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-l8ksqxqxvhll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-w78qcb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-w78qcb\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x39bf51e7ba70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-l8ksqxqxvhll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-w78qcb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"697309d8df16b27753c6b0671e6b829487d36943157b8d477fabeff5b5aea0bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c929d59edb7567fa76ef70a100f78549e067b5211d3fa8ccbe6bb931bf8ae3bc","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x6bee01cdac0\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-6td8ds\" resourcegroup=\"rg-np-skip-minor-6td8ds-d2dvtglcn2l2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6td8ds-d2dvtglcn2l2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6td8ds/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x6bee01cdac0\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-6td8ds\" resourcegroup=\"rg-np-skip-minor-6td8ds-d2dvtglcn2l2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6td8ds-d2dvtglcn2l2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6td8ds/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"70c71db71d474d600450c9194163e989c401e548db8d5bd50fecdaf31d395fde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"5bb1d3c7e546810e558daf11bf222544e4133eca40ba362ffb48b7b1423ad104","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x177ce192fa20\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-z4pwtb\" resourcegroup=\"rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-z4pwtb/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x177ce192fa20\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-z4pwtb\" resourcegroup=\"rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-z4pwtb/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"76315c6f7dd1f60bcbe1892d73ecb0525546f62b7ea9847f1227a533b1e818d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"a6f43bb89c4cb869e470fa6b8f415931a9ffca73b7a099b8d5d5fdb88da93883","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x143a1310b240\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-rc8d5d\" resourcegroup=\"rg-np-version-upgrade-rc8d5d-w2t72x2zshww\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rc8d5d-w2t72x2zshww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-rc8d5d/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x143a1310b240\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-rc8d5d\" resourcegroup=\"rg-np-version-upgrade-rc8d5d-w2t72x2zshww\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rc8d5d-w2t72x2zshww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-rc8d5d/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"797713a3285ba10190c71961df753255912c25d035e20b13982126106b0a503b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"1a76286ada45eadee61eadc06e02438d66934a94c715501b79fa55ea647992ba","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w2grzz\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x30efed875560\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-52rk6bpxrc9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-w2grzz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w2grzz\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x30efed875560\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-52rk6bpxrc9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-w2grzz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7983ab0a2937940d5d7eb2f22a2828fbb126c6caf70b70fb774ba5fce7e5e5b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"0d1682a2cfbb5f2e590b6b31ccc2bdd35b4fb6efe346454c0a0465b4a4c377dc","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x148c3b6e47b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-tvg824p96xln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x148c3b6e47b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-tvg824p96xln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8ccac32d6425dc4004b10368b148c88ca26fe55f64ca66f8d0fb3f5fb30b3b0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"d46d14f8b1e583f77549aa7ebed41d05be17337007389372864730bed2620d7f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x95d711c6660\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fqfljmzwnn7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x95d711c6660\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fqfljmzwnn7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8f411c621c298058ef598a08dced662516f7c8f3bcf1292c64ecac83f75d34b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"6db71a6e85dcae0f4cf2fff89193035c7c77e8c6a0d75a696207a46ff9701f52","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x16c8ce6032a0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-4rvxbgx64wn7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4rvxbgx64wn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x16c8ce6032a0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-4rvxbgx64wn7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4rvxbgx64wn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"91e2e721a9f657bba3bd5ae357910154adbe8144e749f0d75326e400b961e6e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c17fe792fb98b2d6a545cd9ddbd5a4c5a6ed15360c664292bd1e61a9ec602478","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-2bw2nd\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x3e7c72aaef90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-h5wrz4q9tk28/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-2bw2nd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-2bw2nd\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x3e7c72aaef90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-h5wrz4q9tk28/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-2bw2nd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a71b9f02e0ad43c68c7551662ed1ed112c875525e01a7091c48f1ff1ce969b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"919aee9941c508698346026ce9e0051090805691e68054cfc4acc83d6c2fb7d3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-4c45hm\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1c31b53c8540\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7jn7lrmbjhwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-4c45hm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-4c45hm\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1c31b53c8540\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7jn7lrmbjhwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-4c45hm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b264154137a8aa708b1d0300fc426cbfe7ce683d9f7bcb626e6aa0fb56875b78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"9af200308adea58cb657b883d457e0c1c8c4b68df178d893c6ee77bf70f5c283","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18d8f5386060\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-xvr7qd\" resourcegroup=\"rg-np-noedge-xvr7qd-fdgmzp5mlmp2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xvr7qd-fdgmzp5mlmp2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-xvr7qd/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18d8f5386060\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-xvr7qd\" resourcegroup=\"rg-np-noedge-xvr7qd-fdgmzp5mlmp2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xvr7qd-fdgmzp5mlmp2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-xvr7qd/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b7cf4cd93fde135d4224e2903f03f401dd58d99b926a5fc63f1ab451e7ae56fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c3e02910fd51fc69091b5e63232a999211acc1001c4336ec9c326557d10e1d3a","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xa952f808ee0\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-x47s9n\" resourcegroup=\"rg-np-downgrade-x47s9n-djktngm9cgxt\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-x47s9n-djktngm9cgxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-x47s9n/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xa952f808ee0\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-x47s9n\" resourcegroup=\"rg-np-downgrade-x47s9n-djktngm9cgxt\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-x47s9n-djktngm9cgxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-x47s9n/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bce71be1223ae0a8de29cc53e81adc386c029a8381a038187dc4e9073274c3b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"3906d99cd657415581fb6ba0344b0053b97db85bb68b94222e88264810fdd66b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-95bqxj\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1aa712d69710\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b5g72gdsstzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95bqxj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-95bqxj\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1aa712d69710\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b5g72gdsstzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95bqxj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c18b1ae301dc8e50ce54f88d113b3e1f662ccbbe2b68de097fc69a2ae8096cf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"567f7755bb3e007fee5b222ee1274b9e42bd25b0c9f732e516cf41596ecc8fab","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0cb099f2c0\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-cvfvg5\" resourcegroup=\"rg-np-version-upgrade-cvfvg5-r74zmmc476nw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-cvfvg5-r74zmmc476nw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-cvfvg5/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0cb099f2c0\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-cvfvg5\" resourcegroup=\"rg-np-version-upgrade-cvfvg5-r74zmmc476nw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-cvfvg5-r74zmmc476nw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-cvfvg5/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d314dd0c9769760148bae161c2ecb5dc416965037b20b77b772f1612a9233d42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"44dfa2bda4098c665bbfb261786dda1e03ddea5270c9178a60475577c209a9b5","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x110fe0e8c640\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djddxfxjp2wr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-djddxfxjp2wr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x110fe0e8c640\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djddxfxjp2wr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-djddxfxjp2wr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f220b6975bcdd014353909edeea54ebd1e8241886f7abc9f715ca2f1025dac44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"0cf8bd41b605f3c023454134078e0b29fb24116c4ab84dc44eaed46533bbc5d6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-79cvw2\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c456aecc90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-8lpbkjqgxtmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-79cvw2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-79cvw2\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c456aecc90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-8lpbkjqgxtmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-79cvw2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":22,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":22,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}