{"meta":{"start_date":"2026-03-29","end_date":"2026-03-29","start_at":"2026-03-29T00:00:00Z","end_at":"2026-03-30T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T18:03:49Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":4,"failed_runs":3,"raw_occurrences":27,"matched_occurrences":27,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"e519f1887d9740d59629140c909a7511daaa51821ca3498a363bc02241e26c7f","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-qg8fcc\" in resourcegroup=\"admin-credential-lifecycle-test-zsg6lt\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-qg8fcc in resource group admin-credential-lifecycle-test-zsg6lt, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":24,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":24,"after_last_push_count":24,"trend_counts":[0,0,0,0,0,0,24],"trend_range":"2026-03-23..2026-03-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":10,"prior_last_seen_at":"2026-03-12T07:38:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001508390\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nqc9bn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nqc9bn, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-nqc9bn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nqc9bn, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001392170\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nqc9bn\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001508390\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nqc9bn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nqc9bn, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-nqc9bn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nqc9bn, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001392170\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nqc9bn\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:156]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00129fb00\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2nlfhj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2nlfhj, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-2nlfhj\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2nlfhj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0007a6b60\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2nlfhj\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:156]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00129fb00\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2nlfhj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2nlfhj, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-2nlfhj\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2nlfhj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0007a6b60\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2nlfhj\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001211320\u003e: \n    failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2m22rw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-2m22rw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2m22rw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-2m22rw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00092c660\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-2m22rw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001211320\u003e: \n    failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2m22rw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-2m22rw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2m22rw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-2m22rw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00092c660\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-2m22rw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038260698240258048","occurred_at":"2026-03-29T14:23:14Z","signature_id":"42e7aa42f36e7691bbf60636fc811f21f425dc1f897e614e4d0841f004649bb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038260698240258048","occurred_at":"2026-03-29T14:23:14Z","signature_id":"c195a36ab8f5c8db0d93f016ca003230f9349008d179d5ac3337eb55aa752f45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038260698240258048","occurred_at":"2026-03-29T14:23:14Z","signature_id":"f8c89275bf955f4b3c4ade42c733558799db634b2a945d37c1aeec8ab697e983","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038260698240258048","occurred_at":"2026-03-29T14:23:14Z","signature_id":"ff6293e10f2758d1abe5ad92318ebcbaa8ca122ad20894acf55249a7eaf018a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"00360cc2395b76a42bb9582c1d6605476724a3c5d165d16accd5fa72db2513d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"0d3a21877ea2e9eaa94f54f630225d82b83ef6952c76a421a40dc52a480591c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"3b545fbd7a6918df68d71020ffd0b065236d93cb934a3f72c378b3ecb14d2de2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"3de7ad22d8fe30c2e9477de667af65d9c3a64d634999cacad1a1053156aefdc5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"41c28bc8e352d28d0efe047ee253b5813498d1b2c38976e5fe3333fa3eec1f12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"94609d1b736067e2ae9938fcfd93eb7ae768b96b8b9c03e091449a13c49bc068","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"9e6a96a9a3f39fac45989d465c0adb24930167a51e2661f2d7414669e1086cc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"ac3ef309b9fc43e6f5ce13c621c4fcede5681f21dbfa8244fe5a7887cc96399d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"ae1810961d5c2b630ad17ba8061c355d61ac218fb6e383e608925b340aa252bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"c9dc7eda11dc41075ea64cd35f642c5dd3c36c0b2bc829167b37d575e60f3426","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"d0a71a0510dc4b5486adabaa21f1ca9866d0968fe158ab3b52cbc4d6dae86b9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"df67e06c39b41b18a3a08fc755053723b1e9847e3e763632c266acb9e1b45955","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"f9f51a358fb36bf932060969968906254ad5245a396ae86ab3ab674ab1d30830","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"06a0b4ab1c78a20122b938b34eed6968d44463570ecc8ab37304f8d5fbfdd825","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"604b15d4aa8e4c73b69b01c4bfd450d9b359012496c285bb0c634f3dc8239ae3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"7cc7ccf0796cec9d59e1068d1cfa0c13f0347f6f32b307bef44c5586ae087925","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"b80589f5585e56530d0e1c2cd78f17864b5ab7664640b407b2dc1a8723b403a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"c2ebc4f01a1475212b00456e94156ecf87e6e5bbb7c2778d9a8ac3a53d6d3cfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"c79ee4ba278ae83a1c982d7548f0125e85eb5855b4f51992fd7560dcb87b8a33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038107327839604736","occurred_at":"2026-03-29T04:13:47Z","signature_id":"f1b6fcdd76057a79e7381f9d156ba9e4eb1896dc0df9c6c0790a37c1c8f8b1f7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d2973977438d8e970cb26c45be13f7a3ebd47e52a45ae782e685cf476877173a","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,5,2],"trend_range":"2026-03-23..2026-03-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-09T15:26:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009e28a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-vm728z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-vm728z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0008ed830\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"855\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260329T123708Z:c6998271-44f0-41ca-8da6-1f46b5fe6948\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 29A874DFCC374E0391D1296600C0E257 Ref B: CH1AA2020617027 Ref C: 2026-03-29T12:37:07Z\",\n                    ],\n                    \"Date\": [\n                        \"Sun, 29 Mar 2026 12:37:08 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Request-Id\": [\n                        \"5bfee52a-0937-45f3-82af-d95d755a8729\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/6a20fe18-7658-4e7c-bdbc-1e74dd3ff699\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"c6998271-44f0-41ca-8da6-1f46b5fe6948\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009e27e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"manageme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009e28a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-vm728z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-vm728z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0008ed830\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"855\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260329T123708Z:c6998271-44f0-41ca-8da6-1f46b5fe6948\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 29A874DFCC374E0391D1296600C0E257 Ref B: CH1AA2020617027 Ref C: 2026-03-29T12:37:07Z\",\n                    ],\n                    \"Date\": [\n                        \"Sun, 29 Mar 2026 12:37:08 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Request-Id\": [\n                        \"5bfee52a-0937-45f3-82af-d95d755a8729\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/6a20fe18-7658-4e7c-bdbc-1e74dd3ff699\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"c6998271-44f0-41ca-8da6-1f46b5fe6948\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009e27e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"manageme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009aac00\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-79sq4q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-zskdm8/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-79sq4q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-zskdm8/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0010d7dd0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"560\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"8d6cae0d-8889-4205-aa31-fe38545dd223\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/1101af73-6be3-42a3-a250-ff1e67521ad4\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Sun, 29 Mar 2026 12:37:09 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260329T123709Z:f22db835-02bb-4bfb-9e27-d21a5b4df8a1\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 55288A5C640E4D0B9D2013D93A2A6472 Ref B: CHI261140510031 Ref C: 2026-03-29T12:37:08Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"f22db835-02bb-4bfb-9e27-d21a5b4df8a1\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009aaac0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                   ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009aac00\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-79sq4q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-zskdm8/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-79sq4q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-zskdm8/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0010d7dd0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"560\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"8d6cae0d-8889-4205-aa31-fe38545dd223\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/1101af73-6be3-42a3-a250-ff1e67521ad4\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Sun, 29 Mar 2026 12:37:09 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260329T123709Z:f22db835-02bb-4bfb-9e27-d21a5b4df8a1\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 55288A5C640E4D0B9D2013D93A2A6472 Ref B: CHI261140510031 Ref C: 2026-03-29T12:37:08Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"f22db835-02bb-4bfb-9e27-d21a5b4df8a1\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009aaac0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                   ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"5f1d821a7842fd4d7f5eb06ee1bb1766d9e9c60a2e51f4c81d5faf18018af017","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038212316792623104","occurred_at":"2026-03-29T11:10:59Z","signature_id":"d404f2ef85a9c754cd568553c1c92e91f535b31abf62580940675d496e82a1bf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d0993ee7c174e66d21d6873cdcbb83cae1352c1482345816cea0f68947670c76","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":26,"trend_counts":[0,0,2,3,2,0,1],"trend_range":"2026-03-23..2026-03-29 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-16"],"prior_runs_affected":12,"prior_last_seen_at":"2026-03-18T07:37:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["e/proc.go:466\nruntime.semacquire1(0xc001686608, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000a595f8 sp=0xc000a59590 pc=0x2b3451d\nsync.runtime_SemacquireWaitGroup(0xc000a30c20?, 0x40?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000a59630 sp=0xc000a595f8 pc=0x2b59bae\nsync.(*WaitGroup).Wait(0xc001686600)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000a59658 sp=0xc000a59630 pc=0x2b7d0e5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000548500, {0x7f9abcaaa958, 0xc00160ca50}, {{0xc000daf270, 0x1, 0x1}, 0x34630b8a000, {0x7995009, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:308 +0x273 fp=0xc000a59768 sp=0xc000a59658 pc=0x4fdc233\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000548500, {0x7f9abcaaa958, 0xc00160ca50})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:227 +0x225 fp=0xc000a59888 sp=0xc000a59768 pc=0x4fdbac5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f9abcaaa958?, 0xc00160ca50?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000a598b0 sp=0xc000a59888 pc=0x4fea753\nruntime.call16(0xc0011e87e0, 0xc000091030, 0x0, 0x0, 0x0, 0x10, 0xc000a59b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000a598d0 sp=0xc000a598b0 pc=0x2b5f163\nruntime.reflectcall(0x73e36c0?, 0xc00160ca50?, 0x2?, 0x79c5cd8?, 0x0?, 0x12?, 0x7018be0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000a59910 sp=0xc000a598d0 pc=0x2b679f6\nreflect.Value.call({0x699dce0?, 0xc000091030?, 0xc00000029f?}, {0x7989dfb, 0x4}, {0xc000e30f60, 0x1, 0x2ae7368?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000a59e70 sp=0xc000a59910 pc=0x2b9a9c6\nreflect.Value.Call({0x699dce0?, 0xc000091030?, 0x0?}, {0xc000e30f60?, 0x326f4c8?, 0xc000e00150?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000a59ee8 sp=0xc000a59e70 pc=0x2b99ab9\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x8400760?, 0xc00160ca50?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000a59f88 sp=0xc000a59ee8 pc=0x2fba290\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000a59fe0 sp=0xc000a59f88 pc=0x2fccd5b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000a59fe8 sp=0xc000a59fe0 pc=0x2b60c01\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 104\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 1180 gp=0xc00137efc0 m=nil [select]:\nruntime.gopark(0xc000095ab8?, 0x2?, 0x80?, 0xfa?, 0xc000095aac?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000095938 sp=0xc000095918 pc=0x2b5848e\nruntime.selectgo(0xc000095ab8, 0xc000095aa8, 0xc0016104e0?, 0x0, 0x33570dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000095a78 sp=0xc000095938 pc=0x2b33277\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x83ea848, 0xc000e9f4f0}, 0x79a98af?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000095ae8 sp=0xc000095a78 pc=0x3356836\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x842fc60, {0x83ea848, 0xc000e9f4f0}, 0xc00095ec50)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x732 fp=0xc000095de8 sp=0xc000095ae8 pc=0x3792ed2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x83ea848, 0xc000e9f4a0}, 0xc001997380, {0xc000d69100, 0x1a}, {0xc0017bae70, 0x18}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:201 +0x218 fp=0xc000095ef0 sp=0xc000095de8 pc=0x4fbdf38\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:349 +0x94 fp=0xc000095f78 sp=0xc000095ef0 pc=0x4fc0534\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc000095fe0 sp=0xc000095f78 pc=0x3874450\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2b60c01\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1178\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2ad7e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffe140e4e5c\nrbp    0x7ffe140e4e10\nrsp    0x7ffe140e4dc8\nr8     0x0\nr9     0x0\nr10    0x432\nr11    0x246\nr12    0x7ffe140e4ea0\nr13    0x1\nr14    0xb57b700\nr15    0x7\nrip    0x2ad7e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\ne/proc.go:466\nruntime.semacquire1(0xc001686608, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000a595f8 sp=0xc000a59590 pc=0x2b3451d\nsync.runtime_SemacquireWaitGroup(0xc000a30c20?, 0x40?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000a59630 sp=0xc000a595f8 pc=0x2b59bae\nsync.(*WaitGroup).Wait(0xc001686600)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000a59658 sp=0xc000a59630 pc=0x2b7d0e5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000548500, {0x7f9abcaaa958, 0xc00160ca50}, {{0xc000daf270, 0x1, 0x1}, 0x34630b8a000, {0x7995009, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:308 +0x273 fp=0xc000a59768 sp=0xc000a59658 pc=0x4fdc233\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000548500, {0x7f9abcaaa958, 0xc00160ca50})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:227 +0x225 fp=0xc000a59888 sp=0xc000a59768 pc=0x4fdbac5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f9abcaaa958?, 0xc00160ca50?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000a598b0 sp=0xc000a59888 pc=0x4fea753\nruntime.call16(0xc0011e87e0, 0xc000091030, 0x0, 0x0, 0x0, 0x10, 0xc000a59b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000a598d0 sp=0xc000a598b0 pc=0x2b5f163\nruntime.reflectcall(0x73e36c0?, 0xc00160ca50?, 0x2?, 0x79c5cd8?, 0x0?, 0x12?, 0x7018be0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000a59910 sp=0xc000a598d0 pc=0x2b679f6\nreflect.Value.call({0x699dce0?, 0xc000091030?, 0xc00000029f?}, {0x7989dfb, 0x4}, {0xc000e30f60, 0x1, 0x2ae7368?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000a59e70 sp=0xc000a59910 pc=0x2b9a9c6\nreflect.Value.Call({0x699dce0?, 0xc000091030?, 0x0?}, {0xc000e30f60?, 0x326f4c8?, 0xc000e00150?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000a59ee8 sp=0xc000a59e70 pc=0x2b99ab9\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x8400760?, 0xc00160ca50?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000a59f88 sp=0xc000a59ee8 pc=0x2fba290\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000a59fe0 sp=0xc000a59f88 pc=0x2fccd5b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000a59fe8 sp=0xc000a59fe0 pc=0x2b60c01\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 104\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 1180 gp=0xc00137efc0 m=nil [select]:\nruntime.gopark(0xc000095ab8?, 0x2?, 0x80?, 0xfa?, 0xc000095aac?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000095938 sp=0xc000095918 pc=0x2b5848e\nruntime.selectgo(0xc000095ab8, 0xc000095aa8, 0xc0016104e0?, 0x0, 0x33570dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000095a78 sp=0xc000095938 pc=0x2b33277\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x83ea848, 0xc000e9f4f0}, 0x79a98af?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000095ae8 sp=0xc000095a78 pc=0x3356836\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x842fc60, {0x83ea848, 0xc000e9f4f0}, 0xc00095ec50)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x732 fp=0xc000095de8 sp=0xc000095ae8 pc=0x3792ed2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster({0x83ea848, 0xc000e9f4a0}, 0xc001997380, {0xc000d69100, 0x1a}, {0xc0017bae70, 0x18}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:201 +0x218 fp=0xc000095ef0 sp=0xc000095de8 pc=0x4fbdf38\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:349 +0x94 fp=0xc000095f78 sp=0xc000095ef0 pc=0x4fc0534\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc000095fe0 sp=0xc000095f78 pc=0x3874450\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2b60c01\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1178\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2ad7e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffe140e4e5c\nrbp    0x7ffe140e4e10\nrsp    0x7ffe140e4dc8\nr8     0x0\nr9     0x0\nr10    0x432\nr11    0x246\nr12    0x7ffe140e4ea0\nr13    0x1\nr14    0xb57b700\nr15    0x7\nrip    0x2ad7e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2038260698240258048","occurred_at":"2026-03-29T14:23:14Z","signature_id":"372efcf123cccb89061cb1c5cbc3d60e4d6d3ad9467d7807a672d17a2f062792","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}