{"meta":{"start_date":"2026-03-12","end_date":"2026-03-12","start_at":"2026-03-12T00:00:00Z","end_at":"2026-03-13T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T17:35:36Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":1,"raw_occurrences":13,"matched_occurrences":13,"runs_affected":1},"rows":[{"environment":"stg","failure_pattern_id":"09d4ef692462a27c00db576d91310eaddda0a56e4545ba064bef74d9acf68c26","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message node pool \u003cnodepool\u003e '4.20.15' must not be greater than Control Plane version '4.20.8'; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-03-06..2026-03-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:120]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005cc900\u003e: \n    failed to create NodePool ea-np-1: failed starting nodepool creation \"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-n6gfpg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6gfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/nodePools/ea-np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool ea-np-1: failed starting nodepool creation \\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-n6gfpg\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6gfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/nodePools/ea-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0005cc860\u003e{\n            msg: \"failed starting nodepool creation \\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-n6gfpg\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6gfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/nodePools/ea-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000711f50\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:47:33 GMT\",\n                        ],\n                        \"Content-Length\": [\"170\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/bf30885a-b80a-44ff-abbe-06ccd35c12ab\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"553745a4-16f3-4cc2-9ef6-d9750ec2411f\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"507\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"9cd59126-3823-4c22-9fc8-76319bb69c4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T234733Z:9cd59126-3823-4c22-9fc8-76319bb69c4f\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:120]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005cc900\u003e: \n    failed to create NodePool ea-np-1: failed starting nodepool creation \"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-n6gfpg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6gfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/nodePools/ea-np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool ea-np-1: failed starting nodepool creation \\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-n6gfpg\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6gfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/nodePools/ea-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0005cc860\u003e{\n            msg: \"failed starting nodepool creation \\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-n6gfpg\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6gfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/nodePools/ea-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000711f50\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:47:33 GMT\",\n                        ],\n                        \"Content-Length\": [\"170\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/bf30885a-b80a-44ff-abbe-06ccd35c12ab\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"553745a4-16f3-4cc2-9ef6-d9750ec2411f\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"507\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"9cd59126-3823-4c22-9fc8-76319bb69c4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T234733Z:9cd59126-3823-4c22-9fc8-76319bb69c4f\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001702100\u003e: \n    failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-v6s5x5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v6s5x5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-v6s5x5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v6s5x5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0017020e0\u003e{\n            msg: \"failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-v6s5x5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v6s5x5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001700540\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6e43f07a-31da-4d85-a701-8c274872558c\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:48:20 GMT\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"1b69d406-2782-4679-9835-a29957ea680e\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"89980359-f898-4b86-aa59-08d8fdbd6860\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"330\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T234821Z:89980359-f898-4b86-aa59-08d8fdbd6860\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 874B815FAAAB466C92DC038D43FC87AD Ref B: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001702100\u003e: \n    failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-v6s5x5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v6s5x5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-v6s5x5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v6s5x5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0017020e0\u003e{\n            msg: \"failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-v6s5x5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v6s5x5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001700540\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6e43f07a-31da-4d85-a701-8c274872558c\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:48:20 GMT\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"1b69d406-2782-4679-9835-a29957ea680e\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"89980359-f898-4b86-aa59-08d8fdbd6860\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"330\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T234821Z:89980359-f898-4b86-aa59-08d8fdbd6860\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 874B815FAAAB466C92DC038D43FC87AD Ref B: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e38220\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-flx585\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-flx585/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-flx585\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-flx585/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e38200\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-flx585\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-flx585/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00084f3e0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8c49472b-65b6-4d72-895b-d553524b74c1\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: D78F8656FC4043FD9F218D19BD1ABC81 Ref B: CHI261140503031 Ref C: 2026-03-12T23:47:23Z\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"37723390-0bd1-44e6-b398-d2a71cfcb423\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2998\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:47:24 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"170\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/17b7b904-f4ea-4faa-a399-2df6fd4e5bb7\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"198\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T234724Z:8c49472b-65b6-4d72-895b-d553524b74c1\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e38220\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-flx585\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-flx585/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-flx585\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-flx585/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e38200\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-flx585\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-flx585/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00084f3e0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8c49472b-65b6-4d72-895b-d553524b74c1\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: D78F8656FC4043FD9F218D19BD1ABC81 Ref B: CHI261140503031 Ref C: 2026-03-12T23:47:23Z\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"37723390-0bd1-44e6-b398-d2a71cfcb423\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2998\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:47:24 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"170\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/17b7b904-f4ea-4faa-a399-2df6fd4e5bb7\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"198\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T234724Z:8c49472b-65b6-4d72-895b-d553524b74c1\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"012d1c9df63ef35c08ff3788ad3c1eef0d1c196ddeefba52f4a16e653fe2de35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"07fa01b55c46bfccc69e1e07650b8b962b73bcb063f516c6c564cfede9eacde9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"26e6e63da42c4c6b8ca7e98a3c4e0e0bea401162cccb8bfba490e549312a1cfd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"3962f196c4718c705b3128ba848e188f6f8587e69166d96a3c14e9fc4ee7f8f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"3ec40a9a5180e30e12ace09ac7e8b3d53c48181c2de60ab14cb11bc4b0c5d104","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"4fd9b8d8f442f42c8fa9aa59d2ae2742b865f4dcdc1995a111bfb2ff7d783b2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"60367320f70cce9f7e29b86e121c4d3e4f752ee7cb0b2a757ad68c8bf7a05679","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"a5623345eed8ef8abd937aae8ad3a5633eb531dd167c806e6306325e47881ef0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"c53f4b843a2b5224c44b1e4c0ba4f670d284eac6848d5983c4fe9b9cda5a9350","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"eabe4a2b0352b4f9f680348c6a6e4c33fb81e3be30322e4d10047889573b5b0b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-f2sqg7\" in resourcegroup=\"admin-credential-lifecycle-test-xbdmbq\" to finish deleting: context canceled","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-03-06..2026-03-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a12330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-f2sqg7\" in resourcegroup=\"admin-credential-lifecycle-test-xbdmbq\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00063c1c0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-f2sqg7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xbdmbq\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00063c1a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-f2sqg7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xbdmbq\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00063c180\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-f2sqg7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xbdmbq\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a12330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-f2sqg7\" in resourcegroup=\"admin-credential-lifecycle-test-xbdmbq\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00063c1c0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-f2sqg7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xbdmbq\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00063c1a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-f2sqg7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xbdmbq\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00063c180\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-f2sqg7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xbdmbq\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00045a750\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-jsm5nn\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00052b920\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-jsm5nn\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00052b8a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-jsm5nn\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00052b880\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-jsm5nn\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00045a750\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-jsm5nn\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00052b920\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-jsm5nn\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00052b8a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-jsm5nn\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00052b880\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-jsm5nn\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"131d275a728bf9b3c8e57373aea9ff784c287a48898f2d538da7b23ba78ac490","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"fa9d99bc838f7c21e8ca0a4998bdbfb6505dd2d3a154fe6d82233a5ac4bd1918","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"604745dc1eb894c4ac98add4f5ede2030b2008c5ab8972455c0f282bc06425ad","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-06..2026-03-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000c0f0f8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0002fc120\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc000662dd0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000c0f0f8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0002fc120\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc000662dd0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032228084094078976","occurred_at":"2026-03-12T22:51:46Z","signature_id":"6d52a9e8a7f8076045f1781c1f9378599862e85acc61b146e09fabc155403e9c","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}