{"meta":{"start_date":"2026-09-16","end_date":"2026-09-16","start_at":"2026-09-16T00:00:00Z","end_at":"2026-09-17T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T13:21:13Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":13,"matched_occurrences":12,"runs_affected":1},"rows":[{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-09-10..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3799930be8939ca48534a28b262ff33328138426e7cec1b9fb2793e6f4a95899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"474ef928e4bfaa163ff38e770c6435689354cdc8821b9a05ec4272d8ad60c351","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"7a42cc1723b1b8bf9843f1aa12dd8baec4448f6917d7f084b3a4b154baa63afa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"8be57f60e7cd91c124c72d6d01098c6e1b8b853355d51b79f6b1bd55c240efdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"bd4c07c42b25870c0f1baaecfa45c3c668afd2f7347cf758e572515c16e421fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-10..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3d8d555b219d0ed8c9661c956fba32429ade34240dfb89932e7b6f838ec71e56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"51c0a7fe19269071bf4a86bc72b032196ba0e6e4e142e092ab1f3aca5f282e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a3bcc0cf85a56ec12c072c420e0931c744e3a86ca315e7af1b9dd0b787fa5b91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"c259628d57d4675c02d0130322edaa375c878722b676fb3290dbafba51543aa4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-10..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121ee40\u003e: \n    failed to create NodePool np-krkxx8: failed waiting for nodepool=\"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\" in resourcegroup=\"rg-nightly-4-23-7l9rdmzm7f2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"name\": \"4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:37:25.694607859Z\",\n      \"endTime\": \"2026-09-16T23:56:27.737971571Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121ee40\u003e: \n    failed to create NodePool np-krkxx8: failed waiting for nodepool=\"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\" in resourcegroup=\"rg-nightly-4-23-7l9rdmzm7f2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"name\": \"4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:37:25.694607859Z\",\n      \"endTime\": \"2026-09-16T23:56:27.737971571Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"18ab22781017136dd055b840d502183c7edf1caa41f67dcadf9fd0a7857c8081","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5c1e65a74237eb97d61c283de75bcde39935ce10b1226f1bc067c35315e03a3d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-10..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5c2b3fa28b82ccf14825b20d7135a7cb3d0c7f8bdbad597c1b9eba4c000e0333","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-10..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"77617baa5bf45330c8ac3ac9717b66793245c58ebcc17c8a3e357cf43d6f0bf5","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}