{"meta":{"start_date":"2026-09-27","end_date":"2026-09-27","start_at":"2026-09-27T00:00:00Z","end_at":"2026-09-28T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T09:15:27Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":30,"failed_runs":27,"raw_occurrences":68,"matched_occurrences":67,"runs_affected":26},"rows":[{"environment":"dev","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":26.666666666666668,"anchor_occurrences":8,"after_last_push_count":9,"trend_counts":[0,0,0,0,6,5,8],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":19,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T23:20:54Z\n  Ended:       2026-09-27T23:44:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j9480704-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j9480704-mgmt-1\nNamespace: ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:05:16Z\n  Ended:       2026-09-27T22:39:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2557696-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2557696-mgmt-2\nNamespace: ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T20:30:25Z\n  Ended:       2026-09-27T21:00:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T20:30:25Z\n  Ended:       2026-09-27T20:55:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"141dcb4a7bce83ddae2534a50b4037182ee14743f68296b8e0c644526062ca81","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","occurred_at":"2026-09-27T21:02:53Z","signature_id":"a4af37eae3845736c3bca8e48388195a8cf0fb39b9828c0799328f4f81338760","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","occurred_at":"2026-09-27T19:25:36Z","signature_id":"80970b397031adb31bef780349e99ab505e3f2fd73c9134ca7e696f1e795203c","pr_number":7200,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"426761ff9070cec28a5322afd36d3627552c3d22962de9e435913509b416aa8d","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","signature_id":"44a63252c6af03437dca2a2cd16633fffc635048800be390cd8ed0b750e906c9","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","signature_id":"35bed69d107d0e48eeae3149c2b40959f5c177d8606a49c69653469eb84a80bc","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","signature_id":"bdc02d01d20a4113becec172cb87ce69502ed7be7f54534e201fa4c1be3941cb","pr_number":7197,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"6493d354b73970d4e764ed97c72ddea6bbaa29dc9a60565e51aa0b12f093243d","pr_number":7174,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":26.666666666666668,"anchor_occurrences":8,"after_last_push_count":7,"trend_counts":[0,0,0,0,5,4,8],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":17,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T23:23:56Z\n  Ended:       2026-09-28T00:09:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j9480704-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j9480704-mgmt-1\nNamespace: ocm-arohcpci01-2t3l81mo181qkj3h3jcrvelvtdhmn5ea-a0v8t8r0i7d7k3n","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:08:16Z\n  Ended:       2026-09-27T23:04:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2557696-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2557696-mgmt-2\nNamespace: ocm-arohcpci01-2t3k4ukpcu7g8mb4j4of7t2s9p7chht2-y0z1h5i8m3z9p7t","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T20:33:23Z\n  Ended:       2026-09-27T21:25:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ipgo5l7305q17rs36aqaiigj71lve-agg-cluster\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T20:33:23Z\n  Ended:       2026-09-27T21:20:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0960384-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0960384-mgmt-1\nNamespace: ocm-arohcpci01-2t3ip3j1bcpr5hi565ci3j9dn87t9gfa-g7x2p2x3z9l1o5m"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"419f780bdbaa43f4ef8f4ecb0e33396b8bfad0e38a30159c1ff2da81cde2f1a7","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","occurred_at":"2026-09-27T21:02:53Z","signature_id":"d6d997e13699f3708b8da2df30da87216f574a5d894c86f3487f4f6d6d9ec091","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","occurred_at":"2026-09-27T19:25:36Z","signature_id":"ba0d797453f4acd2d3294ab03019f050a2ea64a58bfda6df78da0690b02a2c50","pr_number":7200,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"5afb798429b94f5c90e2fc1387a8558369362de6b6d0c8a51b724a0353effcc8","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","signature_id":"8d21629c67d82189d892875373e45a37e10d571da501c0c8f7adab114af4262c","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","signature_id":"5e5cd04319eed373eaaf914f96f5fe47f4a0a57f723cd12e6afda7269f6982ed","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","signature_id":"6a3860a8f043002633cbc86fed6af04ae8f3209a874cc185b949e92160d28645","pr_number":7197,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"92318fcfac24b6233ea2adc5b233527abdcb8533537932bdc38c9c6b9dd86ffc","pr_number":7174,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":8,"after_last_push_count":1,"trend_counts":[0,0,0,0,2,1,8],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de2a20\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tcmg5wddbpw5\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"name\": \"75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.92972128Z\",\n      \"endTime\": \"2026-09-27T22:26:36.766928511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de2a20\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tcmg5wddbpw5\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"name\": \"75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.92972128Z\",\n      \"endTime\": \"2026-09-27T22:26:36.766928511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f767a0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tlzxr8jmz7kk\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"name\": \"4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.95420119Z\",\n      \"endTime\": \"2026-09-27T22:26:17.083195861Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f767a0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tlzxr8jmz7kk\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"name\": \"4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.95420119Z\",\n      \"endTime\": \"2026-09-27T22:26:17.083195861Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e260\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-hhxkjbgbq9fb\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"name\": \"3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.777648405Z\",\n      \"endTime\": \"2026-09-27T22:26:17.06664737Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e260\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-hhxkjbgbq9fb\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"name\": \"3710ea21-b4a2-479d-839b-69b7c8bb9004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.777648405Z\",\n      \"endTime\": \"2026-09-27T22:26:17.06664737Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"0170db780c4df4adf7bb8a89ac92e967f775ba081325c4cac8d09effb77a4c27","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"046677b799dbe94e2abdd61ad6c306ffc70f772329a705fd183eeddfe68167fe","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"056f5aac48261bf6a11c23178e7053c153d8a77c8cd07b47c6d8f44988ca9be3","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"14017b17f411363983778e1786bd28771dd63d57ce07d0aef5decd8c20e45c89","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"21db66633adda02d93dd92446231c93c6316598ee599dce2a1a985be29893e32","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"3edaa5633377ee32cc909ec3f497506e56f88cc918442b0177acc1481df58f76","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"8fc8ffdd4987b337260088b62ea7c21d838b773930ca05d6c0bce06df0b64bba","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"fe1e6109eb86fcb5bef47d66924fe8d64130a64ca2fed35ead60f414517a7f41","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":7,"runs_affected":4,"failed_runs":4,"run_impact_percent":13.333333333333334,"anchor_occurrences":7,"after_last_push_count":4,"trend_counts":[0,0,0,0,6,2,7],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6d380\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-4tsj8j\" in resourcegroup=\"rg-np-noedge-4tsj8j-67tl4d7r7ztd\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"name\": \"36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:24:00.446121961Z\",\n      \"endTime\": \"2026-09-27T23:43:03.337836309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6d380\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-4tsj8j\" in resourcegroup=\"rg-np-noedge-4tsj8j-67tl4d7r7ztd\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"name\": \"36f4f936-b9bd-4488-b93c-580f6594d3e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:24:00.446121961Z\",\n      \"endTime\": \"2026-09-27T23:43:03.337836309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4tsj8j-67tl4d7r7ztd-np-ne-4tsj8j--managed/providers/Microsoft.Compute/virtualMachines/y7o9d9s7b3m0t4u-npnoedge-4-20-pwn57-gs8w5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060efe0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wz5nqx5brl4f\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"name\": \"3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:20:03.595087025Z\",\n      \"endTime\": \"2026-09-27T23:39:03.372293439Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-dq426: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-wz5nqx5brl4f--managed/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Network/networkInterfaces/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060efe0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wz5nqx5brl4f\" to finish creating: GET https://rp.j9480704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"name\": \"3e30fc72-f8f0-4109-846c-1ad5795f6f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:20:03.595087025Z\",\n      \"endTime\": \"2026-09-27T23:39:03.372293439Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-dq426: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-wz5nqx5brl4f--managed/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Network/networkInterfaces/u0j2v3g1y2f6r0v-np-1-6pwv6-dq426-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wz5nqx5brl4f--managed/providers/Microsoft.Compute/virtualMachines/u0j2v3g1y2f6r0v-np-1-6pwv6-b4jpl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa8060\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kcdxbmxql2zw\" to finish creating: GET https://rp.j8939648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"name\": \"4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:35:22.462990652Z\",\n      \"endTime\": \"2026-09-27T20:54:31.807536747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa8060\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kcdxbmxql2zw\" to finish creating: GET https://rp.j8939648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"name\": \"4c589deb-304d-4351-af73-1056e66b3ff1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:35:22.462990652Z\",\n      \"endTime\": \"2026-09-27T20:54:31.807536747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kcdxbmxql2zw--managed/providers/Microsoft.Compute/virtualMachines/b7y1q5d8i0e4m4p-autoscale-noaz-2mzsj-hlnq4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"01e7808196f8b35484085dcce2391cd7ff226b123e9d686fab532297c48cc3ba","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"0edcc1c21b78bd45a5dcd2f988466a7ec1ce2a2761e3091d1504343efd9bb710","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7201/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291389238939648","occurred_at":"2026-09-27T19:25:37Z","signature_id":"eea917c119fb496a73e78c0ccb01ffe97195fb9bbf05312d263c9d161c01f38e","pr_number":7201,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","signature_id":"0263dc178beb748cec45d68667416149811aa14e2aff4db55448a33e6e63b5c7","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","signature_id":"0ef5b19fd1cdd2f1706442a99609b4839aa21fe049bf6ef3221a44afe9300590","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","signature_id":"f11b81cb85742fe5ff7b9b961ba857992240995905349698dcbcbd8852080bc6","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"673f5d1216f0f7b060fd38c36ae28fe2306c5398d3ffd8fc404991258602f0b7","pr_number":7174,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":20,"anchor_occurrences":6,"after_last_push_count":68,"trend_counts":[20,21,22,24,30,5,6],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":165,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:00:15Z\n  Ended:       2026-09-27T18:06:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1132032-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1132032-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T16:57:42Z\n  Ended:       2026-09-27T17:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5652992-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5652992-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T08:29:34Z\n  Ended:       2026-09-27T08:35:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0172672-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0172672-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","occurred_at":"2026-09-27T16:28:23Z","signature_id":"d18c749f3722a30d0857576cb4260959235b0821801cb56556539b905f76002f","pr_number":7199,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"f9c445cff424013d845cfdec607b448ab1198b54444d73458cb4dab5656a8299","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","signature_id":"430c9f14f7bfbeb1599ee7c1c8f52ce30424b0cc5c567fd877a045c6cf4ab795","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","occurred_at":"2026-09-27T04:51:29Z","signature_id":"c24afa2c3a89b1db96f561ea699ffc16b6ecf7c1b6c7babc01f986b2e2f32c4a","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"1a89df4c7c180c8de0615a9daa2555665cfe23d16df3d8e06611837f74f549ed","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070538413477888","occurred_at":"2026-09-27T04:48:02Z","signature_id":"84013e20db593dd86936f009da2bff278404d255c4473789a9a242cf0c678207","pr_number":7194,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":9,"trend_counts":[0,0,0,0,5,3,5],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":13,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:25:52Z\n  Ended:       2026-09-27T19:47:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j1590528-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1590528-mgmt-1\nNamespace: ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:14:22Z\n  Ended:       2026-09-27T17:26:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j5652992-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j5652992-mgmt-2\nNamespace: ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T08:05:14Z\n  Ended:       2026-09-27T08:31:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0172672-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0172672-mgmt-1\nNamespace: ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"893ce1f9fd178961fa552c9c4ed69700c8bd2c12d9b189640fba68210e5e2004","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"a3273e0d1794b7c8a67f767205c53f73d87ec1d167083395847a4aaed230b76f","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","signature_id":"9d4a34fd40b7a4768d4e4eabe4a4ec42749a225062d6842b119ed23e64fabb54","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"0bafa741a68390f4b6bf980e8d38b5adb37e713e43141fb95567a5f642ddcfdd","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","signature_id":"c79fa012a0b071c6064bdd63844fb3a758f0f6a2bc109b1ebc88b8a973287f91","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":8,"trend_counts":[0,0,0,0,4,3,5],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T19:28:50Z\n  Ended:       2026-09-27T20:12:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j1590528-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j1590528-mgmt-1\nNamespace: ocm-arohcpci01-2t3hqd26j6rv21sf05e07diagki1b9ln-f5s3p9a3j1f3z1q","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:21:23Z\n  Ended:       2026-09-27T17:51:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j5652992-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j5652992-mgmt-2\nNamespace: ocm-arohcpci01-2t3fn15tq7pskns61germc68tebdv165-z2y0q7d5f0a7u7e","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T08:08:19Z\n  Ended:       2026-09-27T08:56:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0172672-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0172672-mgmt-1\nNamespace: ocm-arohcpci01-2t37ra42qcsvctongdu6ee8ia8ttj4b9-p6i6n4h8u0d8b6r"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"191a6f3923b9adfd1af697a8104aad4e301e37d24962d749afe729bbd97e8f3a","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"66132c0ef28f02375a925f830a63f89a139e8421a3d4bf3c8d7b497dcbd25498","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","signature_id":"0ae2d2740aeab65c0927cc3b7b9751bc45171ae3ffe4aada6c1848ef04948ef5","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"c1605d1431fa38cffcc889bc3b160bbc5879105dd589db8bc2b28a285e29f8b6","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","signature_id":"33e8f05c23ee905d13012921f05b90827ce9b99ceb5852cf28e896b69118e778","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":13.333333333333334,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,2,0,1,0,4,4],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":21,"prior_last_seen_at":"2026-09-19T02:41:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104208646610096128","occurred_at":"2026-09-27T13:56:50Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104124987630686208","occurred_at":"2026-09-27T08:24:24Z","signature_id":"97240d8885a0bdfc86afa503aac43fee59507e22b5387aeb628e8d3e06faee04","pr_number":7105,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":51,"trend_counts":[2,2,2,4,6,1,2],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":103,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 13 time(s) (6 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:06:28Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T22:06:29Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-27T22:07:30Z\n  Ended:       2026-09-27T22:48:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-27T22:07:30Z\n  Ended:       2026-09-27T22:48:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-27T22:09:27Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-27T22:10:30Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd-0 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:37:18Z\n  Ended:       2026-09-27T17:43:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1132032-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3g642vb3ovmhl4qiumvarv9qifaek4-c6e1u6h4n5k3a7j\", pod=\"router-5584fc994b-5x9hp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3g642vb3ovmhl4qiumvarv9qifaek4-c6e1u6h4n5k3a7j/router-5584fc994b-5x9hp has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"7c6c23658687695d208580c0620f85e276d1932e76e62b017338962064930246","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","occurred_at":"2026-09-27T16:28:23Z","signature_id":"bd2bb1667dcc46addf88c81f320c44761ca728d893e15805a3bd4ad0673a41d2","pr_number":7199,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[3,6,4,2,12,0,2],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":39,"prior_last_seen_at":"2026-09-20T16:10:57Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T14:40:20Z\n  Ended:       2026-09-27T14:57:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5467008-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T01:21:42Z\n  Ended:       2026-09-27T01:35:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8776064-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T01:22:43Z\n  Ended:       2026-09-27T01:37:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8776064-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7197,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104002487378776064","occurred_at":"2026-09-27T00:17:38Z","signature_id":"194da11ceda26cf004e70e30811c51ee51ab4977a0682ac50eca40b8c54129c3","pr_number":7185,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":9,"trend_counts":[0,4,1,4,0,0,2],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":18,"prior_last_seen_at":"2026-09-20T12:02:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000676bd0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000676bd0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002177290\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002177290\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104191454321577984","occurred_at":"2026-09-27T12:48:31Z","signature_id":"5926710b9fa5f78e3a02a2c8e840a39699891ffa25494a7619c4e511a8bddcd0","pr_number":7196,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","occurred_at":"2026-09-27T04:51:29Z","signature_id":"f0dd05bcedb68800765b2105c3e810ca2e4be249bca3b12d32e008a16b7acfdb","pr_number":7187,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[1,0,2,2,3,0,2],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-19T10:22:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000f972a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-t7rmzspt8s2k\" to finish creating: GET https://rp.j6124032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"name\": \"0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:26:25.730010902Z\",\n      \"endTime\": \"2026-09-27T20:45:26.626992422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-t7rmzspt8s2k--managed/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-t7rmzspt8s2k--managed/providers/Microsoft.Network/networkInterfaces/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' seconds after additiona... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000f972a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-t7rmzspt8s2k\" to finish creating: GET https://rp.j6124032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"name\": \"0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:26:25.730010902Z\",\n      \"endTime\": \"2026-09-27T20:45:26.626992422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-t7rmzspt8s2k--managed/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-t7rmzspt8s2k--managed/providers/Microsoft.Network/networkInterfaces/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' seconds after additiona... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00089fe00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp.j5652992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"name\": \"6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T17:01:59.070822701Z\",\n      \"endTime\": \"2026-09-27T17:21:03.436757642Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00089fe00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp.j5652992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"name\": \"6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T17:01:59.070822701Z\",\n      \"endTime\": \"2026-09-27T17:21:03.436757642Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104290856126124032","occurred_at":"2026-09-27T19:23:30Z","signature_id":"a7885b2a5379fa2f9992b69c22b756f6bcde94e94685985bf119094349859833","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"2d46f58172df00b26b08a908114c7df93ee1d42789e568fd90c7a7b65ba15046","pr_number":7198,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,2,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a2980\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-5fskdvpkxfwb\" to finish creating: GET https://rp.j7860224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"name\": \"2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:48:02.556455592Z\",\n      \"endTime\": \"2026-09-27T06:07:09.276038359Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2: Failed: interfaces failed to create or update. err: failed to get existing resource admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/providers/Microsoft.Network/networkInterfaces/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a2980\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-5fskdvpkxfwb\" to finish creating: GET https://rp.j7860224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"name\": \"2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:48:02.556455592Z\",\n      \"endTime\": \"2026-09-27T06:07:09.276038359Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2: Failed: interfaces failed to create or update. err: failed to get existing resource admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/providers/Microsoft.Network/networkInterfaces/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","signature_id":"ec4257f3d7d834907b3d94066f4a938edbf0824cf3b15c1781d4153ac05d1938","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"56fd2feac0bf2fd733880f128445ced8b465f2cded0a3ea3fc3aa25b8699b45c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Failed; provider Microsoft.Kusto","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kusto-grant-ingest (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kusto-grant-ingest (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-27T04:55:22.142Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 description=\"Step kusto-grant-ingest\\n  Kind: ARM\\n  Template: templates/kusto-grant-ingest.bicep\\n  Parameters: configurations/kusto-mgmt-ingest.tmpl.bicepparam\"\ntime=2026-09-27T04:55:22.662Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2\ntime=2026-09-27T04:55:23.741Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 deployment=2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8656896-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80\ntime=2026-09-27T04:59:54.839Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8656896-mgmt-2/providers/Microsoft.Resources/deployments/2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80/operationStatuses/08584111219623969246\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/grantKusto-hcp-ja2yqt7pclopi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-kusto-us/providers/Microsoft.Kusto/clusters/hcp-dev-us-2/databases/HostedControlPlaneLogs/principalAssignments/grant-b6fd691c-fba0-50f9-bb53-13af017771ec\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Internal Server Error\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104062724328656896","occurred_at":"2026-09-27T04:16:59Z","signature_id":"24248bfb9172326e331f0bc70fbaea3432445ca771da2ecd7e9aa55cd2f3ee9a","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,1,0,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-20T04:02:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:133]: failed to get admin REST config for cluster \"cluster-candidate-4-21-cscwdf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001673c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-cscwdf\" in resourcegroup=\"rg-candidate-4-21-xbwd2trtds6r\" to finish getting creds: GET https://rp.j6653824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:133]: failed to get admin REST config for cluster \"cluster-candidate-4-21-cscwdf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001673c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-cscwdf\" in resourcegroup=\"rg-candidate-4-21-xbwd2trtds6r\" to finish getting creds: GET https://rp.j6653824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","signature_id":"05ebaa9488bd93eeb90e405c068beae1c93f0836a7ac6badc62d7e0b3eadd4c5","pr_number":7183,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,0,2,1,6,1,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-18T18:24:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029105a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029105a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104123485520400384","occurred_at":"2026-09-27T08:18:26Z","signature_id":"9b8aa1049a27cadfcf00f24cced762a2cb52851eacbb3b0c5ad6f8e5a3f78f90","pr_number":7193,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[4,3,2,1,3,1,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":30,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 12 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-27T22:19:32Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-27T22:19:32Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-27T22:20:28Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-27T22:20:28Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"e00c24569b7fef8b6c9f87bbc44d7468ad1c94ae60292dc7180f5c34825d2871","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:49:46Z\n  Ended:       2026-09-27T18:56:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j8175232-svc\", component=\"fleet\", name=\"nodepoolcontroller\", region=\"westus3\", severity=\"warning\"\n  Description: Fleet controller workqueue nodepoolcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104264194718175232","occurred_at":"2026-09-27T17:37:33Z","signature_id":"b1312384d9b2a53ea723368157d1cd9705e0e9f23816fc33749b5b53bb2fe8fb","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,3,2,4,1,2,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":21,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 8 time(s) (1 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T14:26:50Z\n  Ended:       2026-09-27T14:32:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5453056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3d20g44qhuupsokqk6foneks2ohpk7-agg-cluster\", pod=\"kube-apiserver-857c58cf4-hl9vw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3d20g44qhuupsokqk6foneks2ohpk7-agg-cluster/kube-apiserver-857c58cf4-hl9vw has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104196521745453056","occurred_at":"2026-09-27T13:08:39Z","signature_id":"325bf50ff4030e0a9e49b5267b69a16cbb66d95e03425b035183cf4895a04dc3","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5a32867416c248246850f053eb447f1cb93d8fba2bae3455ee41d0153f99c5c0","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": read tcp \u003cip\u003e-\u003e\u003cip\u003e: read: connection reset by peer","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000858480\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000858480\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"e8a205541114f6cec3b425e2ad1adb4dc2be321629c4a93557e1f3b4a5460bb4","pr_number":7178,"after_last_push_of_merged_pr":false}]}]}]}