{"meta":{"date":"2026-10-01","timezone":"UTC","generated_at":"2026-10-29T21:14:30Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105806972916862976","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-01T23:48:00Z","started_at":"2026-10-01T23:48:00Z","completed_at":"2026-10-02T02:10:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T02:31:11.784426075Z","timing_metadata_checked_at":"2026-10-02T02:31:11.784426075Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T02:31:11.787402065Z","region_metadata_checked_at":"2026-10-02T02:31:11.787402065Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T19:47:05Z","started_at":"2026-10-01T19:47:05Z","completed_at":"2026-10-01T22:18:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T22:28:49.582676757Z","timing_metadata_checked_at":"2026-10-01T22:28:49.582676757Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T22:28:49.58837874Z","region_metadata_checked_at":"2026-10-01T22:28:49.58837874Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6b183935159f0d52f2c697f494ad2cb956c0f00f11e24f897bba981d654f70fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","occurred_at":"2026-10-01T19:47:05Z","failed_at":"e2e","signature_id":"a7f9c67af898546f71a808604e7639e1ebc1c0ed5391f8286fb71f6bda1d5873","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000941160\u003e: \n failed to create NodePool np-jv9hvk: failed waiting for nodepool=\"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\" in resourcegroup=\"rg-candidate-4-20-6nfp6mj4gt68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n \"name\": \"29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T20:46:24.926645724Z\",\n \"endTime\": \"2026-10-01T21:05:28.967720321Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000941160\u003e: \n failed to create NodePool np-jv9hvk: failed waiting for nodepool=\"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\" in resourcegroup=\"rg-candidate-4-20-6nfp6mj4gt68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n \"name\": \"29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T20:46:24.926645724Z\",\n \"endTime\": \"2026-10-01T21:05:28.967720321Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c6b8dae8893a41575bd688a67356818f4d9e6388e776c10bcb5b98511c637459","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","occurred_at":"2026-10-01T19:47:05Z","failed_at":"e2e","signature_id":"24417d5ef077cde226947335837e6d950a92fb685e28c57995b46f4508d8e565","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c9e80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fxmrv96q875w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\",\n \"name\": \"a1048b90-244a-4c5a-ba89-e1135a68a269\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T20:23:38.274618399Z\",\n \"endTime\": \"2026-10-01T20:42:38.037012934Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012c9e80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fxmrv96q875w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\",\n \"name\": \"a1048b90-244a-4c5a-ba89-e1135a68a269\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T20:23:38.274618399Z\",\n \"endTime\": \"2026-10-01T20:42:38.037012934Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105698247119998976","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-01T16:35:58Z","started_at":"2026-10-01T16:35:58Z","completed_at":"2026-10-01T18:59:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T19:00:20.202577378Z","timing_metadata_checked_at":"2026-10-01T19:00:20.202577378Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T19:00:20.206347266Z","region_metadata_checked_at":"2026-10-01T19:00:20.206347266Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T10:32:01Z","started_at":"2026-10-01T10:32:01Z","completed_at":"2026-10-01T12:48:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T12:58:59.269749862Z","timing_metadata_checked_at":"2026-10-01T12:58:59.269749862Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T12:58:59.277819036Z","region_metadata_checked_at":"2026-10-01T12:58:59.277819036Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"2f0fc6f2a81697acec4031d037774c93056c9cb98e86b8297a87045b195ffe10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"af5e31a50e667e6789ed1a04349bf851557b9188ce2f9b1e9df3b6b675fccd7c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-282pdp\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000fe7f50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-qszf5tc8smsd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-282pdp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-282pdp\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000fe7f50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-qszf5tc8smsd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-282pdp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"5fe39dcb2cee87a73235205259a516d2ae32eaa371fb9cf36496786be2cf859c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"09ad1536393e5c2e93d734e6e4a7baabe569c00b7a346b81e194a1dc526d6c84","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-fpkhgg\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013695c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-x2qm64ntnf4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-fpkhgg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-fpkhgg\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013695c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-x2qm64ntnf4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-fpkhgg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"6198db3d9c66f474d0a7db12745dc44bc315b4ca4d123d00b0eb8899e60b6fb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"e47528e4415d38ba10698d98e1ae80c6cfa1a4632c798890c9c25df89d4ae02f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-zhcqgm\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f2be90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-bkxc6qpdr2kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-zhcqgm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-zhcqgm\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f2be90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-bkxc6qpdr2kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-zhcqgm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"7727ac737b8c97b9eb3a6aefbf1be66692e39fc5bc6d5e762e293a92a914d68e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"fae23cdbef205ba19e74bef0e349a708114fea21f0bfce452cbd47ff26764f2f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-ncq7tv\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014ac6f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-jnrgf69jtww7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-ncq7tv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-ncq7tv\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014ac6f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-jnrgf69jtww7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-ncq7tv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"7941ab9f85f14f438fad741b42399a5e64c7372ff8913aa1850d7faf555ffaf3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"53d5243f6499b6bbb1f41dd260ea791225c721f0e2219068f0049f16fa9044cd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5jgcvl\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00156e510\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-qqh9mh2dgv5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5jgcvl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5jgcvl\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00156e510\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-qqh9mh2dgv5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5jgcvl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"7b3b060cccd6bd6ec8cbac371fe7bdbf989936aa350cc3691e820aeb0a41a6e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"8928cf54b630d0e6361d33389106d1e4b00e5b4fdda04424c7b1b3c9b344e25a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-cznf64\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018c4c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-z6bx6clhw9nr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-cznf64\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-cznf64\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018c4c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-z6bx6clhw9nr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-cznf64\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"98c4e4f4ad6517c1901c03052a037d343150c2cf5a74efc0f33bb5c1d6740b51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"1e70068ca83e3da2caa181c14b566633dee772461c6bbef7e3b3c3b11c58ede2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5lppxg\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000c77c20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-9qkw2hc8j64k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5lppxg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5lppxg\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000c77c20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-9qkw2hc8j64k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5lppxg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"a50f45f0327dc73dab1ec435e20d88e4b9725f255125ec1620ee9c144abf0e91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"881e0d266cdbc141cde714bffad14ec487d883af4eafed8f23bce4c4912d543e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-rtvfwf\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012da990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-zzffnqvsn5bk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-rtvfwf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-rtvfwf\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012da990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-zzffnqvsn5bk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-rtvfwf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"a9c911fb77bd53fd5ed23459b6c12d4a8cea036eba2706df769addfa00f70d9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"93276e8c2e659949c04804f296318e19a7261db922d78b2b7a491384b13155c9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-59lhdf\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b51650\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-zncx9w9pdbrq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-59lhdf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-59lhdf\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b51650\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-zncx9w9pdbrq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-59lhdf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"f9402448b1afaa8146ba0b95ff266bdf5eccbc4678d7522aba74fd2c20bd0692","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","failed_at":"e2e","signature_id":"ee5d11d2818efb7c3c4ed23a4e01d5e6ab980563ce83dc358792a3f97e2d1f1f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-5gft95\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015d78c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-8c7gqffnfmhg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-5gft95\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-5gft95\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015d78c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-8c7gqffnfmhg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-5gft95\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T07:19:36Z","started_at":"2026-10-01T07:19:36Z","completed_at":"2026-10-01T09:26:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T09:41:17.467693184Z","timing_metadata_checked_at":"2026-10-01T09:41:17.467693184Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T09:41:17.470831574Z","region_metadata_checked_at":"2026-10-01T09:41:17.470831574Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"08ac35a05cb721304349889db73830d4a7ef7a737a1544155a6f0f1d0ee1a107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"ad3a6f95d691523f560574aa964dea297931db1f1651191d1fcc28b85649a55a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-tvg8ls\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001483050\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-8ptjnkq2z7df/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-tvg8ls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-tvg8ls\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001483050\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-8ptjnkq2z7df/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-tvg8ls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"1732f574930d170c3a0a650fe852e310c9965a13fbe9ce5b6428fc725e52d58f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"ffa6512d457f2bdedb8df8b3e86503ab5c120285bf88c97a42b1b851b87335e3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-fbsgr5\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00153e5d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rfwzkqxm7bss/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-fbsgr5\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-fbsgr5\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00153e5d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rfwzkqxm7bss/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-fbsgr5\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"2874df5170aecb746f38dcde1db974c6bc5c2f081017d44285e63c9ed9924b5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"a5ad9b8fa8feb1257e28216d87d1a08d738224a79e57266bf4690f58f661ad42","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-gfc5ks\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b4b2f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-2k4s9c5c5gt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-gfc5ks\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-gfc5ks\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b4b2f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-2k4s9c5c5gt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-gfc5ks\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"5208eef2089879af744c3cd512bb4fe23cba933e24b61c26b8ea8015051eb88b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"4b7a489ef3ea65128a354e82da4f4e1de6bc22ca63444681abe780ccc7ddf4d4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-skgz2q\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014816b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-j4gwfrfwllb5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-skgz2q\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-skgz2q\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014816b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-j4gwfrfwllb5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-skgz2q\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"72776b5f17c05e617c38aef2cb92591a2eb3082a72585d01ff52acf290938eac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"a23758f9b4de17fd810383f0e57b42b4438075c36125aa9b6c712edcb5ea75a5","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-97ngnf\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009a0ab0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-l7xhqzrtl9sq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-97ngnf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-97ngnf\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009a0ab0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-l7xhqzrtl9sq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-97ngnf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"c463df6b59235f9a4a98f61f5377edf7a538c5fda2c9e6bd1d0b8b7db079bbde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"9ed87e9e9315982414d4d556eba0d0bea3967a50c31ea93eff6a36c532a8eb2a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-84kjqn\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00159a510\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-9tsgcnv7kf56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-84kjqn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-84kjqn\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00159a510\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-9tsgcnv7kf56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-84kjqn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"ca9e59a3e206554ac8f96ae9ad2276a8ab41424ceb1e4d55dfab18a638fab067","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"f393133e89a119909f1414ed37aedeb249de63da9461ad1d5feb1022be0d1b09","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-6vqlpz\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001492180\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6954mm5k46dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-6vqlpz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-6vqlpz\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001492180\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6954mm5k46dr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-6vqlpz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"cd2123221f9294a52feed8eb872fc1186cc7c1f736022b33e4e9967e549f3a8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"3f239f77699bb84da05b2be013b74a288bb4538f8ac6f15ec4ea749a60735ad4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-gkw8ql\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011fba70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-lzvgwqsshhgh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-gkw8ql\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-gkw8ql\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011fba70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-lzvgwqsshhgh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-gkw8ql\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"d9647e51143e36b5b671834d9d82a7be02e8d23947cb8b85cdc38ec63df0679e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"b42e83d1c1e188b5a53a98054e2e876e0d0416f2e4658e9a37a24c36e12854fe","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-qkh6np\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000af7500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-jqv2lkbmlggl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-qkh6np\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-qkh6np\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000af7500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-jqv2lkbmlggl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-qkh6np\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"ddbb57cf3f17d145e7dd385d8e82aab000ef7280993491e80a159435e0398ee3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","failed_at":"e2e","signature_id":"3a4c53c8b8ed0f4e3a8c8625b638982616ccea702a79f9698e9303da79a15f67","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5xch77\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001152bd0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-869wg248knm4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5xch77\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5xch77\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001152bd0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-869wg248knm4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5xch77\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T01:33:40Z","started_at":"2026-10-01T01:33:40Z","completed_at":"2026-10-01T03:41:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T03:44:31.916152159Z","timing_metadata_checked_at":"2026-10-01T03:44:31.916152159Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T03:44:31.91933715Z","region_metadata_checked_at":"2026-10-01T03:44:31.91933715Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"0f9f9a0a34c1e575c047c1c6c6d9b02d42b00f50d9b1c5876f0e927eab5cdd15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"eda6d7fa1b7ee822da5e9598ea4cc8fe8832dbc41c8d88d803855760b7f64988","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-rngq5s\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ccdb30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-mlc5mpzcvj8q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-rngq5s\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-rngq5s\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ccdb30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-mlc5mpzcvj8q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-rngq5s\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"14d1824e93b3c8505ccd6c9ed7ccf8f2849c84bff62dafe9cbdef725c8be42d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"652af8ac6d7caa46ada925ea05e4b4f678aa2fa90a5c6290737d7b660b7f2248","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-zx7mxd\" to follow minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018b1e90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-gvghfmdnj9kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-zx7mxd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-zx7mxd\" to follow minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018b1e90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-gvghfmdnj9kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-zx7mxd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"2bd4e1f6e743c0a2e1b640f1b683d92bf1b39a2497ba764cf587beee8943d5fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"0ec303987eddb2fdc7951010bee64e64f19b2795c36b27fa9b6c16d511039f6d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-22-tkvcqf\" to follow minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00155e240\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-9l9l89pgr4m5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-tkvcqf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-22-tkvcqf\" to follow minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00155e240\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-9l9l89pgr4m5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-tkvcqf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"4185cf6f7a97c8740a5a52ac0d940f951f4da30cacaa7783eddd5e022d44ea41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"10b16b7588fc77933a91ee23f48c974f59b3cf6c7796512f1986ba476c2fdcba","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-4p7wkl\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001587980\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-2dqnlv5bkbch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-4p7wkl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-4p7wkl\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001587980\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-2dqnlv5bkbch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-4p7wkl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"6611afc924f75a554973e5ebbaf488a5273a37272236f169bb940ea74ecad093","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"44b6707750ec56079daaa0398aa289b9339df5e0faca1b38d9f04663d954b374","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109d0c0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s7p6hptqhbvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\",\n \"name\": \"357a31b9-a87e-4b62-a8b5-196b066234f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T02:58:38.669148771Z\",\n \"endTime\": \"2026-10-01T03:17:44.618360897Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109d0c0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s7p6hptqhbvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\",\n \"name\": \"357a31b9-a87e-4b62-a8b5-196b066234f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T02:58:38.669148771Z\",\n \"endTime\": \"2026-10-01T03:17:44.618360897Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b3f3e0a11bd87e1d792b79fd0f26a19cb61cf8deaa26da50a5b798fb45c47433","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e1dbb817a499ecaa7243ca5e5308c3f3f5e727824dfaa5ffc5a66d4658ecc034","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"de5390b7de0f9fb6b5b999f0840b119c4cfcea4837064e22c1a37c81aa8301d8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-tsd828\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ab0180\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-qpv6d29gz4jl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-tsd828\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-tsd828\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ab0180\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-qpv6d29gz4jl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-tsd828\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"e3c8e4c98749f3c3f02b0ef9bc9193a3e1e9589a86887da9b95167296c19f0e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"bc6beae7f23eaeb00f646e9afe501ded4abd7941cafe250a38a63603361caf1e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-5-0-q5mklf\" to follow minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00118f7a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-rcvrxff9wvxp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-q5mklf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-5-0-q5mklf\" to follow minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00118f7a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-rcvrxff9wvxp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-q5mklf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"e4fd7349e13531c8370b1656b14c155e2d6121a23c8f991e35fe952246f0aaeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"a2c9210be7e28a9733e941546fc75b632c067fbedccb9d0a6c5fc389f9a2e48f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gswqqb\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0005f0a80\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nmmbsqvg94ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gswqqb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gswqqb\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0005f0a80\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nmmbsqvg94ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gswqqb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"e93f4ebc23fe3524c6d7b9acbe72b50282fb8e9e291d8d9747646253508da258","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"a8535460c6d3cd45df9df1883152d24842155b835833878a66f7b2d3ba6aaa58","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-vnllsd\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015a4150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5lmngc5rw784/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-vnllsd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-vnllsd\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015a4150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5lmngc5rw784/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-vnllsd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"e9bfd3b89b4a93d328de0ab03fbce64ac736926b8ed20f99dc5f2de5e045a383","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"e33b49f25d1ec8838fb3fb2248feed144429268d006904054aadad3f41b2c4ef","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00057dcc0\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jspphct49kmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\",\n \"name\": \"4a191606-d1f9-418e-b590-0307f5435adb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T01:58:13.545499376Z\",\n \"endTime\": \"2026-10-01T02:17:18.498756095Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00057dcc0\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jspphct49kmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\",\n \"name\": \"4a191606-d1f9-418e-b590-0307f5435adb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T01:58:13.545499376Z\",\n \"endTime\": \"2026-10-01T02:17:18.498756095Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ecf5f9363d9e61e88476ddb993df609655b9230bc2cb58619b6d84b794de93ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"535d9b5810530eb8fa93d005fe5342bc6c5f76b1ac4619d0de9e457c08ad121a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-20-2crl7w\" to follow minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015c2c00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-bdf2955zlsb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2crl7w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-20-2crl7w\" to follow minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015c2c00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-bdf2955zlsb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2crl7w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"fe6d4228db3fb4d437d6d691cbf34a96ba5b861e81f0efd14ac874055642a7d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","failed_at":"e2e","signature_id":"bc99803c563848f8ef2751f1a43b94eaeabe40a9b89129e706b11fea21a2a78d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-4jz6tz\" to follow minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a1c2d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-wjhccp827grc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-4jz6tz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-4jz6tz\" to follow minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a1c2d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-wjhccp827grc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-4jz6tz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","b3f3e0a11bd87e1d792b79fd0f26a19cb61cf8deaa26da50a5b798fb45c47433","d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}