{"meta":{"start_date":"2026-03-16","end_date":"2026-03-16","start_at":"2026-03-16T00:00:00Z","end_at":"2026-03-17T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T19:51:25Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":23,"matched_occurrences":23,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,0,9],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-07T02:00:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00155d560\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\"cluster-zstream-4-20-vrd9tg\" in resourcegroup=\"rg-zstream-upgrade-4-20-vppdwr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a12410\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00155d530\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a12410\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00155d560\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\"cluster-zstream-4-20-vrd9tg\" in resourcegroup=\"rg-zstream-upgrade-4-20-vppdwr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a12410\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00155d530\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a12410\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ddc570\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-fsqvnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013818e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ddc540\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013818e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ddc570\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-fsqvnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013818e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ddc540\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013818e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00144c090\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-l82rz5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000aed740\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00144c060\u003e{\n                msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000aed740\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00144c090\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-l82rz5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000aed740\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00144c060\u003e{\n                msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000aed740\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"08e328dc1dfd1fb161e5ee5fbeb36a333526cb48a4a6251392be2ebade86d839","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"1eb3805a5fe71ec50edd4d69c587ec8350ab24f91e053ebfd61eded6cdf1d4ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"3b352a408296d3a0251d7a6d14862960baca6761c1863c44d92551e765866899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"488befe4edc4450b2b46b4d02c1793f8c3dd4e75a7c8db29f6ea6f12b2d9673b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"5aeff6efa147f0a4f4455f0d12d6e118c4fd736fa1d304c8919d5fa6389a770e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"a3917a20697f535d483e2f50596dc25ed08eb3cecf016c87a084b621b064671c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"d96df068ad434e9a0e2966e4bb5122791524d6797a32adea9a72c57456b99c47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"e84f3d91e58a4ddeb3430bb8931aee0419956b91f0597cc4dde0c7443f9de91f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"f84db3c137cad46a28c3e54119e47a5ebf7698ff6bc2244e20e068bd0436bdd4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":7,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-04T11:18:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00095c4c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-psrj8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"name\": \"b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:54:04.320816036Z\",\n      \"endTime\": \"2026-03-16T05:54:48.517209203Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00095c4a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000974690\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"39e1d21e-9cf9-413f-8981-bd67e2dcc0da\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6c4b46e2-9a40-4232-9e7e-8cba90b18fb2\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/57085293-a6f4-4a48-a611-c5bc14532134\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00095c4c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-psrj8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"name\": \"b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:54:04.320816036Z\",\n      \"endTime\": \"2026-03-16T05:54:48.517209203Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00095c4a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000974690\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"39e1d21e-9cf9-413f-8981-bd67e2dcc0da\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6c4b46e2-9a40-4232-9e7e-8cba90b18fb2\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/57085293-a6f4-4a48-a611-c5bc14532134\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088aac0\u003e: \n    failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\"ea-listnpt7gm\" in resourcegroup=\"ea-list-zq4lft\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"name\": \"09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T06:00:58.543305109Z\",\n      \"endTime\": \"2026-03-16T06:01:21.462955449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088aaa0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00077bbc0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/381463b8-de5e-41cf-a1d6-b149c681d8c1\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T060129Z:a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Msedge-Ref\": [\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088aac0\u003e: \n    failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\"ea-listnpt7gm\" in resourcegroup=\"ea-list-zq4lft\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"name\": \"09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T06:00:58.543305109Z\",\n      \"endTime\": \"2026-03-16T06:01:21.462955449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088aaa0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00077bbc0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/381463b8-de5e-41cf-a1d6-b149c681d8c1\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T060129Z:a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Msedge-Ref\": [\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000695540\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vgmwdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"name\": \"8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:58:36.120361686Z\",\n      \"endTime\": \"2026-03-16T05:59:25.865971075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000695500\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000adeed0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"20c01661-0afe-48d2-8f91-e191ffbc139d\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055926Z:6229d6a4-a060-4bcd-8adb-767c2b1c2524\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/f93d9d56-0328-4fdd-b9c9-60f270da1dcd\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=3153...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000695540\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vgmwdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"name\": \"8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:58:36.120361686Z\",\n      \"endTime\": \"2026-03-16T05:59:25.865971075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000695500\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000adeed0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"20c01661-0afe-48d2-8f91-e191ffbc139d\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055926Z:6229d6a4-a060-4bcd-8adb-767c2b1c2524\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/f93d9d56-0328-4fdd-b9c9-60f270da1dcd\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=3153...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"0c2eda5743cefd6dcfe9ecf740d39893dbf60d284ed5aac300adeb234f564f8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"0e70a576c756dcd421b0a13e1846e4a5a70d8e4db210c128bf73cd9a2927613f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"3e0e267bf5b593318dc8ae60ef2869ec7a0775117cd2322f89b963a63a68c41f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"420b91d3c81d83d07724defc494257d5a7fd2f05d9cb9b50a3d99e9924d26de0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"6970bcc652bfcfa5b97b4fc669aea33f82002482e6faf25e9f132a1466275393","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"994949bc78b1800725718804f81191e64a987b1448eba49bcdbdb8afecca0b76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"f4dcf9151c86716fa85f7d2354f36e247bbe355f6542caad32346c5877430eb7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-05T07:27:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"893592381c30302c0ad58ee9d8f6dc46d73953b616076e02ee3cc8d0f8a1abef","failure_pattern":"ERROR CODE: AuthorizationFailed","search_query":"ERROR CODE: AuthorizationFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001504fc0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster-7hz8zk\" in resourcegroup=\"customer-rg-clsh4k\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001504fa0\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014bb1a0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T060848Z:06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"569\"],\n                        \"Content-Type\": [\n                            \"applic...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001504fc0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster-7hz8zk\" in resourcegroup=\"customer-rg-clsh4k\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001504fa0\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014bb1a0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T060848Z:06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"569\"],\n                        \"Content-Type\": [\n                            \"applic...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"f5fb0b3b4bff165498e2d9ddacd8e71b63b31ca295b435d5432bfc3cf1966be7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"eff4ac5ab820367904c1fb3e78845caba542fd4825dfa1c42a1fdcf6260aa246","failure_pattern":"ERROR CODE: AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: AuthorizationFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008f2c20\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rkr4r8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008f2c00\u003e{\n            msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014046f0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"523\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:50:31 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T055032Z:2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008f2c20\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rkr4r8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008f2c00\u003e{\n            msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014046f0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"523\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:50:31 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T055032Z:2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"db956ec4e40e35159a8af8607f32da52079e787390ee2e9a142f0fef2f1112c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3efdb49df868badec71fe7b3172881c5f208bc6b8873fb2baedf7ca51b8593a4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004993c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-lqkgx2\" in resourcegroup=\"external-auth-cluster-rhfph2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000499380\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000baa6f0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004993c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-lqkgx2\" in resourcegroup=\"external-auth-cluster-rhfph2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000499380\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000baa6f0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"a10960aad13ed9377a917637cf1712dc7c65ca2dd34b3f121b55ef5f9935f1e0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1ab4a070713c2799384044a1c8b07d256e44b43cd64be86c9bc5395d9df5c28e","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00093d060\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-5pp8qh\" in resourcegroup=\"np-autoscaling-l7brcq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00093d020\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008efc50\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: ED37EB9E1FA645FA954684A176851865 Ref B: CH1AA2020620045 Ref C: 2026-03-16T06:05:39Z\",\n                        ],\n                        \"Date\": [\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00093d060\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-5pp8qh\" in resourcegroup=\"np-autoscaling-l7brcq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00093d020\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008efc50\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: ED37EB9E1FA645FA954684A176851865 Ref B: CH1AA2020620045 Ref C: 2026-03-16T06:05:39Z\",\n                        ],\n                        \"Date\": [\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"aa224806ca603d56173bc8c4551b5efd8e98c6c2cdc24d4ee7cb97d7e3e4866e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6461ea09e6946cd19f6c887ff25cabbd85071eb5cb02d829439f92ba37e09b36","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000760480\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-wk5cgh\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvvbjq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000760460\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012bd110\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"839\"],\n                 ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000760480\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-wk5cgh\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvvbjq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000760460\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012bd110\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"839\"],\n                 ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"27453cdf92c9666ede34c49454a77bb28518b6f6a37d519d92e216ae52bf28bc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3abd48de63dd3956140a30196e0f3dc44998cdba01a694484355c666029b9eeb","failure_pattern":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","search_query":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-10..2026-03-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.007s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001039e40\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    {\n        s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.007s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001039e40\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    {\n        s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033362659054718976","occurred_at":"2026-03-16T02:00:10Z","signature_id":"69b6d65c2a309f09c806b140114cf9bcb0384b2a18f675177e552539baa0bdd1","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}