{"meta":{"date":"2026-10-27","timezone":"UTC","generated_at":"2026-10-28T23:38:36Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":113,"failed_runs":79,"runs_with_occurrences":79,"runs_with_failure_pattern_matches":73,"runs_with_unmatched_occurrences":6,"runs_without_occurrence_rows":34,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5244,"pr_state":"open","pr_sha":"d4550ee62563a67a9b6ad205492381d75b0be718","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T23:23:29Z","started_at":"2026-10-27T23:23:29Z","completed_at":"2026-10-28T00:44:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:02:41.327885036Z","timing_metadata_checked_at":"2026-10-28T01:02:41.327885036Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:02:41.334694018Z","region_metadata_checked_at":"2026-10-28T01:02:41.334694018Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"594c06ec7a031c7316a6c093b95d5f227be732e0e2f28bdb324c04b2cfb20341","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","failed_at":"alert","signature_id":"c0be5c770591fc9adef802c76d1c98311b28c10ee1c66cb27508a32677e60094","test_name":"[aro-hcp-observability] [svc] alert userJourneyBackupMetricsAbsent does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T00:19:29Z\n  Ended:       2026-10-28T00:26:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyBackupMetricsAbsent\", cluster=\"ci01-j8270720-mgmt-2\", component=\"slo\", severity=\"3\", slo=\"backup-metrics-absent\"\n  Description: Management cluster ci01-j8270720-mgmt-2 is in the underlay-cluster inventory but has had no successful Velero server metrics scrape for 15m. Check Velero, its PodMonitor, and the management-cluster metrics pipeline. A healthy empty cluster with up=1 is not a failure merely because it has no backup attempts.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T00:20:31Z\n  Ended:       2026-10-28T00:26:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyBackupMetricsAbsent\", cluster=\"ci01-j8270720-mgmt-1\", component=\"slo\", severity=\"3\", slo=\"backup-metrics-absent\"\n  Description: Management cluster ci01-j8270720-mgmt-1 is in the underlay-cluster inventory but has had no successful Velero server metrics scrape for 15m. Check Velero, its PodMonitor, and the management-cluster metrics pipeline. A healthy empty cluster with up=1 is not a failure merely because it has no backup attempts.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"73605e60345d9768cbfbea26978720be4c3942667f138969f6f3eb62f002e92e","failure_pattern":"alert [svc] userJourneyBackupMetricsAbsent fired","search_query":"alert [svc] userJourneyBackupMetricsAbsent fired"}},{"row_id":"b181f65ac87eef9c010eadca77852fb8e5a2600451bac0274a22f10914931ffa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","failed_at":"alert","signature_id":"77f4969f1613eac390c074f1db2e579bea3998e6e11e045a055ac7f824623d4f","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:35:36Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j8270720-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.66:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8270720/providers/microsoft.monitor/accounts/services-j8270720\", microsoft.metrics_include_label=\"service\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-5c8c49b687-k7qwk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:35:36Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j8270720-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.66:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8270720/providers/microsoft.monitor/accounts/services-j8270720\", microsoft.metrics_include_label=\"service\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-5c8c49b687-k7qwk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","failure_pattern":"alert [svc] MaestroServerNoReadyReplicas fired","search_query":"alert [svc] MaestroServerNoReadyReplicas fired"}},{"row_id":"c44666f9b613190e37082cafc31b2bfae26eef21177ec48168e0453a4333104e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","failed_at":"alert","signature_id":"df8c4cf1c0145523d4e17617fc5c14945c699a13507fb64077340994373e075b","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 21 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"aks-istio-system\", pod=\"azure-service-mesh-istio-cni-addon-node-ck87t\", region=\"westus3\", severity=\"warning\"\n  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-ck87t has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-n68qf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-n68qf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-bx6p7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-bx6p7 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-6fggw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-6fggw has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-42508117-vmss000002-249kw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-42508117-vmss000002-249kw has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"aks-istio-system\", pod=\"azure-service-mesh-istio-cni-addon-node-f2bs8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-f2bs8 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-pwgk9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-pwgk9 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-sb9gv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-sb9gv has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-42508117-vmss000001-z2292\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-42508117-vmss000001-z2292 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-sgh48\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-sgh48 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"aks-istio-system\", pod=\"azure-service-mesh-istio-cni-addon-node-jkt84\", region=\"westus3\", severity=\"warning\"\n  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-jkt84 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-42508117-vmss000000-b6bzq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-42508117-vmss000000-b6bzq has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"hubble-relay-58594b5dd7-p7rcj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/hubble-relay-58594b5dd7-p7rcj has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-8696d46cbc-gcq7v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-8696d46cbc-gcq7v has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-8696d46cbc-7tl64\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-8696d46cbc-7tl64 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-6bc6657c6b-nc5dx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod maestro/maestro-6bc6657c6b-nc5dx has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-567b46758d-6wzh8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-567b46758d-6wzh8 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-567b46758d-rsmsf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-567b46758d-rsmsf has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"ocm-cs-db-7594d5ccfc-vd2b7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/ocm-cs-db-7594d5ccfc-vd2b7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-567b46758d-hzqfx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-567b46758d-hzqfx has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-6bc6657c6b-stp2q\", region=\"westus3\", severity=\"warning\"\n  Description: Pod maestro/maestro-6bc6657c6b-stp2q has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"df02d2b32a6d3b27c8b87b0de9e3fc4e773df597c0741d756b6ee92b50c34c88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","failed_at":"provision","signature_id":"ef8a2bb1c5a90008d03e66f07016c90f57a640a5c8bdad13633717d011c3e6b7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-10-28T00:24:38.589Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-28T00:24:39.214Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:80c01b85-e07b-47ba-b2c6-d25569ff69d4 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-ci01-maestro-j8270720.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j8270720] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]] zoneCount:3] fleet:map[namespace:fleet serviceAccount:fleet] image:map[digest:sha256:aba8ed3c2d6e8b0f6a5e17b2f65dba996c5cf673b73a35cc550521873b08c231 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] monitoringApiGroup:monitoring.coreos.com pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:d99261bb-4b6f-46d6-96d4-bae9c86ef284 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-10-28T00:24:39.214Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-28T00:24:39.557Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-10-28T00:24:39.898Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-10-28T00:24:39.898Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-28T00:24:40.025Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-28T00:34:46.993Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"\ntime=2026-10-28T00:34:46.994Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c9b30ffa48a46e2fe98d38066e691b7b2af650635955d57fc11c75ad9e01b688","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["73605e60345d9768cbfbea26978720be4c3942667f138969f6f3eb62f002e92e","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","c9b30ffa48a46e2fe98d38066e691b7b2af650635955d57fc11c75ad9e01b688"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115220467789860864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"closed","pr_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","final_merged_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T23:13:50Z","started_at":"2026-10-27T23:13:50Z","completed_at":"2026-10-28T01:25:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:44:13.977441683Z","timing_metadata_checked_at":"2026-10-28T01:44:13.977441683Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:44:13.980908773Z","region_metadata_checked_at":"2026-10-28T01:44:13.980908773Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115220441911005184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6081,"pr_state":"open","pr_sha":"860fe9f16ea44af6ed07ffbd4b0e9c062b01f3d7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T23:13:46Z","started_at":"2026-10-27T23:13:46Z","completed_at":"2026-10-27T23:27:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:39:01.867658248Z","timing_metadata_checked_at":"2026-10-27T23:39:01.867658248Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-27T23:39:01.878750692Z","region_metadata_checked_at":"2026-10-28T00:41:08.318533826Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bd38b0bedc3e454d122d00493ecbeb5817295326245ca2d2dfb6cf5e9e9e1781","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115220441911005184","occurred_at":"2026-10-27T23:13:46Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115220414635446272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"63ee057fa8457e9b8d70fd6b234bbed2bb83e872","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T23:13:40Z","started_at":"2026-10-27T23:13:40Z","completed_at":"2026-10-28T01:32:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:44:14.121904352Z","timing_metadata_checked_at":"2026-10-28T01:44:14.121904352Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:44:14.125393641Z","region_metadata_checked_at":"2026-10-28T01:44:14.125393641Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115217737012744192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5244,"pr_state":"open","pr_sha":"d4550ee62563a67a9b6ad205492381d75b0be718","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T23:03:01Z","started_at":"2026-10-27T23:03:01Z","completed_at":"2026-10-27T23:18:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:28:22.765730281Z","timing_metadata_checked_at":"2026-10-27T23:28:22.765730281Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-27T23:28:22.771034869Z","region_metadata_checked_at":"2026-10-28T00:31:01.427028266Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"693b45cb0ab548b09afc1e08c50923315c82ff71811c69eb6e1d5a01cb3aecbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115217737012744192","occurred_at":"2026-10-27T23:03:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7436/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115217687146663936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7436,"pr_state":"open","pr_sha":"8759c9fb5b7d83158666e5614c13d6c092220390","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T23:02:50Z","started_at":"2026-10-27T23:02:50Z","completed_at":"2026-10-28T01:16:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:33:49.596859441Z","timing_metadata_checked_at":"2026-10-28T01:33:49.596859441Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:33:49.60084873Z","region_metadata_checked_at":"2026-10-28T01:33:49.60084873Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216183337684992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"fbfc1eab94893088b5d6f4b0d6d6224d411b5d0f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T22:56:51Z","started_at":"2026-10-27T22:56:51Z","completed_at":"2026-10-28T01:37:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:55:03.873968241Z","timing_metadata_checked_at":"2026-10-28T01:55:03.873968241Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:55:03.877825136Z","region_metadata_checked_at":"2026-10-28T01:55:03.877825136Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"c75f751e0f1e506f5d24d0af945a6a88e9263eae6583f1377edb864ad1ed5981","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216183337684992","occurred_at":"2026-10-27T22:56:51Z","failed_at":"e2e","signature_id":"9f72ae7d5870df0c4286d045d51fbd4b7ca3a022b871ba5133e06bc50029740a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x76964b22320\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-x8bmdggt6rx8\" to finish creating: GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"name\": \"589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T00:30:16.899807125Z\",\n      \"endTime\": \"2026-10-28T00:49:22.446308948Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-6hzwl-4vtdh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-x8bmdggt6rx8--managed/ea-cluster-ea-np-1-6hzwl-4vtdh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-6hzwl-4vtdh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n            correlation ID: 10498359-e6e2-4fc8-aa1b-45727688742e\n            client request ID: 1f6fa3eb-c07d-45c3-990e-463cf99216e9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x76964b22320\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-x8bmdggt6rx8\" to finish creating: GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"name\": \"589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T00:30:16.899807125Z\",\n      \"endTime\": \"2026-10-28T00:49:22.446308948Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-6hzwl-4vtdh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-x8bmdggt6rx8--managed/ea-cluster-ea-np-1-6hzwl-4vtdh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-6hzwl-4vtdh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n            correlation ID: 10498359-e6e2-4fc8-aa1b-45727688742e\n            client request ID: 1f6fa3eb-c07d-45c3-990e-463cf99216e9\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3a08509cc4313fdb7de088f97578445c52be0514ee3ea35913c562d65fc01af4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ef5132ee90e0e1eaac4ff2dada0080517634d5c2d7930993c7cb850ba7ba4aa3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216183337684992","occurred_at":"2026-10-27T22:56:51Z","failed_at":"e2e","signature_id":"4b6ad7abc4a4e91efe1a1b87dfaec822083c16eb358fb11b4a36c244a76d6e3c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16b125c17c20\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x16b125c17c00\u003e{\n            observation: \u003c*errors.errorString | 0x16b125e61c20\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x16b125c17b80\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x16b1258ee6e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x16b125c17b20\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x16b127c385d0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16b125c17c20\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x16b125c17c00\u003e{\n            observation: \u003c*errors.errorString | 0x16b125e61c20\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x16b125c17b80\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x16b1258ee6e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x16b125c17b20\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x16b127c385d0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cd80d6b45c8eb9b1b33eb30b789a71289e43bb10694295e955e5ee50ed8fa41","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["3a08509cc4313fdb7de088f97578445c52be0514ee3ea35913c562d65fc01af4","9cd80d6b45c8eb9b1b33eb30b789a71289e43bb10694295e955e5ee50ed8fa41"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216026894340096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7769,"pr_state":"closed","pr_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","final_merged_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T22:56:14Z","started_at":"2026-10-27T22:56:14Z","completed_at":"2026-10-28T01:21:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:33:49.732280854Z","timing_metadata_checked_at":"2026-10-28T01:33:49.732280854Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:33:49.735491245Z","region_metadata_checked_at":"2026-10-28T01:33:49.735491245Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"515666d90387659f24f2035c7463d9486e7825158cfb5d08add4a9f574d7c14b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216026894340096","occurred_at":"2026-10-27T22:56:14Z","failed_at":"e2e","signature_id":"4d308ae304f8d76d5c0a297940b13882945a80c9d3b647dbda29ddcdd9f64ba2","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5f50682dba0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-zj558pxt722v\" to finish creating: GET https://rp.j4340096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a82259f2-e169-4ef9-b443-fe55248e1a5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a82259f2-e169-4ef9-b443-fe55248e1a5a\",\n      \"name\": \"a82259f2-e169-4ef9-b443-fe55248e1a5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T00:07:43.50327074Z\",\n      \"endTime\": \"2026-10-28T00:26:52.986365896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-lrdwb-g7cqn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-zj558pxt722v--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-lrdwb-g7cqn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-lrdwb-ck62b: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-zj558pxt722v--managed/v20260901-np-1-lrdwb-ck62b-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-zj558pxt722v--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-lrdwb-ck62b-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'fd35ac5f-6493-40cb-9131-d669a70114c3' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4340096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a82259f2-e169-4ef9-b443-fe55248e1a5a\n            correlation ID: 3ba28a91-2919-42f2-8746-e40059a09487\n            client request ID: ed3e3d87-8d0e-41b4-b0be-5121fe0a6ea7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5f50682dba0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-zj558pxt722v\" to finish creating: GET https://rp.j4340096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a82259f2-e169-4ef9-b443-fe55248e1a5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a82259f2-e169-4ef9-b443-fe55248e1a5a\",\n      \"name\": \"a82259f2-e169-4ef9-b443-fe55248e1a5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T00:07:43.50327074Z\",\n      \"endTime\": \"2026-10-28T00:26:52.986365896Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-lrdwb-g7cqn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-zj558pxt722v--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-lrdwb-g7cqn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-lrdwb-ck62b: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-zj558pxt722v--managed/v20260901-np-1-lrdwb-ck62b-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-zj558pxt722v--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-lrdwb-ck62b-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'fd35ac5f-6493-40cb-9131-d669a70114c3' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4340096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a82259f2-e169-4ef9-b443-fe55248e1a5a\n            correlation ID: 3ba28a91-2919-42f2-8746-e40059a09487\n            client request ID: ed3e3d87-8d0e-41b4-b0be-5121fe0a6ea7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7792/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115211637647478784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7792,"pr_state":"open","pr_sha":"032c0661b61b2b94d499d37d7c470831329169e0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T22:38:47Z","started_at":"2026-10-27T22:38:47Z","completed_at":"2026-10-28T00:41:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T00:51:48.707117941Z","timing_metadata_checked_at":"2026-10-28T00:51:48.707117941Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T00:51:48.710108032Z","region_metadata_checked_at":"2026-10-28T00:51:48.710108032Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115211604302761984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7739,"pr_state":"closed","pr_sha":"85f478eff30207b95d439e1d26791f75c7ff20f7","final_merged_sha":"85f478eff30207b95d439e1d26791f75c7ff20f7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T22:38:39Z","started_at":"2026-10-27T22:38:39Z","completed_at":"2026-10-28T00:44:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:02:41.563232906Z","timing_metadata_checked_at":"2026-10-28T01:02:41.563232906Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:02:41.566929596Z","region_metadata_checked_at":"2026-10-28T01:02:41.566929596Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7787/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115208311295971328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7787,"pr_state":"closed","pr_sha":"e3706c63dab225fa10d5d48a8e6d7445b8732ced","final_merged_sha":"ffe9a1a535afaeb4ceb215e0f27b578fa90f9ded","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T22:25:34Z","started_at":"2026-10-27T22:25:34Z","completed_at":"2026-10-28T00:39:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T00:51:48.575648242Z","timing_metadata_checked_at":"2026-10-28T00:51:48.575648242Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T00:51:48.579150832Z","region_metadata_checked_at":"2026-10-28T00:51:48.579150832Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6e76893b3a1352a1c3a37385b8770d406a56e3c2b0146ebae58dd8c3c9d5b6cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7787/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115208311295971328","occurred_at":"2026-10-27T22:25:34Z","failed_at":"e2e","signature_id":"8cbd1f8525d7ee50ddb682a6839345188c008a0a6039e30cba962b84e2ffdd23","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x910473e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x91047780d20\u003e{\n            s: \"dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x910473e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x91047780d20\u003e{\n            s: \"dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1591afee4f2df55f5110e8b81c5b4958d76dbcd81b95ec7776997e6bc1915980","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s","search_query":"occurred"}},{"row_id":"90f1aee4c32a356f7eb6f1c42c98b64663eb0d1bf1e2385128f5b0278e62d8d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7787/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115208311295971328","occurred_at":"2026-10-27T22:25:34Z","failed_at":"e2e","signature_id":"8febf95c3a0ca16f0a2a3359fd8ca2fcc7ba552e29fd5c4139ff44661fb4f3d7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-prdbsm\"\nUnexpected error:\n    \u003c*errors.joinError | 0xfaf21d3c618\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xfaf223bc0a0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0xfaf223bc080\u003e{\n                    msg: \"failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0xfaf223bc060\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0xfaf21fc43c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"cluster-np-prdbsm-dxj9f-p58c2\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-prdbsm\"\nUnexpected error:\n    \u003c*errors.joinError | 0xfaf21d3c618\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xfaf223bc0a0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0xfaf223bc080\u003e{\n                    msg: \"failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0xfaf223bc060\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0xfaf21fc43c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"cluster-np-prdbsm-dxj9f-p58c2\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"057c32882f63a3eb320e4cfbfa6ca35e12ecb53508e538854f6156c738380877","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node cluster-np-prdbsm-\u003cid\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-\u003cid\u003e\": nodes \"cluster-np-prdbsm-\u003cid\u003e\" is forbidden","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["057c32882f63a3eb320e4cfbfa6ca35e12ecb53508e538854f6156c738380877","1591afee4f2df55f5110e8b81c5b4958d76dbcd81b95ec7776997e6bc1915980"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6924,"pr_state":"open","pr_sha":"3bbf34616046013a9f4abb722faa0cee0dd81a6c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T22:20:39Z","started_at":"2026-10-27T22:20:39Z","completed_at":"2026-10-28T00:45:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:02:41.445020822Z","timing_metadata_checked_at":"2026-10-28T01:02:41.445020822Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:02:41.448806312Z","region_metadata_checked_at":"2026-10-28T01:02:41.448806312Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"6a3c0139a79bb1fb41a6e8e1c51e2897502787f70f3721fbb994d748c580d31a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","failed_at":"alert","signature_id":"5f52e62e8a632dc6425c669a8d6a48a88e99ab3ee83dc1b80737d8ba7f2924f7","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T23:40:09Z\n  Ended:       2026-10-27T23:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1933056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tneda066vj6kdgdjqh3ub42gu0lckg9-ephemeral-disk\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tneda066vj6kdgdjqh3ub42gu0lckg9-ephemeral-disk/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"952d9a5577001d8075d1df9e79ccee78d10eefca39c8ed388f6c4ee59f5be37b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","failed_at":"e2e","signature_id":"0d6baa6c9c1c38b46fb24aaf5c600e2b1a68d61ebd9a06bf240d594129184dc0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26ee15ea6540\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lz9smvq6kbwg\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"name\": \"ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:28:59.117849892Z\",\n      \"endTime\": \"2026-10-27T23:48:07.430527402Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n            correlation ID: e016f547-ae67-4080-be2a-aa90f089b488\n            client request ID: c52a8689-3c38-4f1f-a275-e48620136ff8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26ee15ea6540\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lz9smvq6kbwg\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"name\": \"ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:28:59.117849892Z\",\n      \"endTime\": \"2026-10-27T23:48:07.430527402Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n            correlation ID: e016f547-ae67-4080-be2a-aa90f089b488\n            client request ID: c52a8689-3c38-4f1f-a275-e48620136ff8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b23b0fdfdd157d4e8c94ca30f28ffe28273f7f126f8dfd41ec44cd3174e770d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","failed_at":"e2e","signature_id":"e39e3016c6f3cedf1cbe0ad7596d7aa46a070ea53b8406bc85c701f66772306e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3591356d60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"name\": \"e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:27:39.482004964Z\",\n      \"endTime\": \"2026-10-27T23:46:40.23395218Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n            correlation ID: 0fae4e0c-a402-45c6-8599-10c6e41f505d\n            client request ID: c1da63b5-9f53-4e68-8989-fc0f3ef1f3cb\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3591356d60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"name\": \"e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:27:39.482004964Z\",\n      \"endTime\": \"2026-10-27T23:46:40.23395218Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n            correlation ID: 0fae4e0c-a402-45c6-8599-10c6e41f505d\n            client request ID: c1da63b5-9f53-4e68-8989-fc0f3ef1f3cb","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp"}},{"row_id":"d1dcdbf3956b510a6d7fa2eafb26cd61fe6d803a65f793c8ec662b83add4ef8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","failed_at":"e2e","signature_id":"c3b990384c16e31bb3ad5ff9f7b85816b1f59b27a4fe9f73c57e1f5d44e83c84","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-n5f5ld\"\nUnexpected error:\n    \u003c*errors.joinError | 0x29f2244004b0\u003e: \n    VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-gkfmqf\" not yet present in zone rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-gkfmqf' does not exist in resource group 'rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n            correlation ID: d271f6f6-6cb4-472d-b6b5-9e1294932892\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-n5f5ld\"\nUnexpected error:\n    \u003c*errors.joinError | 0x29f2244004b0\u003e: \n    VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-gkfmqf\" not yet present in zone rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-gkfmqf' does not exist in resource group 'rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n            correlation ID: d271f6f6-6cb4-472d-b6b5-9e1294932892\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bfef0b8bc199ef07e1bd5837228e606b5ba41f12b4a928e0e7b19c9c7610dda","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-gkfmqf' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}},{"row_id":"eda5f1fc69bc88b092951c21b5b698f8b7f1f60870924d1f3de95208f5a96b4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","failed_at":"e2e","signature_id":"0a0ea7133cf05c130127f02fb8aeee975c38e805f3657e70069a7cb13aced4f0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xb71d4fca080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bqbx2znvzb54\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"name\": \"0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:25:38.076776565Z\",\n      \"endTime\": \"2026-10-27T23:44:40.197052822Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xjlrw-mjpj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xjlrw-mjpj4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xjlrw-7jnb9: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-bqbx2znvzb54--managed/ingress-cert-le-np-1-xjlrw-7jnb9-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-xjlrw-7jnb9-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n            correlation ID: d0096963-5f3a-4ca8-942f-d516adbeb2fc\n            client request ID: 336f75bd-67db-4bb4-b035-81a9154977bb\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xb71d4fca080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bqbx2znvzb54\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"name\": \"0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:25:38.076776565Z\",\n      \"endTime\": \"2026-10-27T23:44:40.197052822Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xjlrw-mjpj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xjlrw-mjpj4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xjlrw-7jnb9: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-bqbx2znvzb54--managed/ingress-cert-le-np-1-xjlrw-7jnb9-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-xjlrw-7jnb9-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n            correlation ID: d0096963-5f3a-4ca8-942f-d516adbeb2fc\n            client request ID: 336f75bd-67db-4bb4-b035-81a9154977bb","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","6bfef0b8bc199ef07e1bd5837228e606b5ba41f12b4a928e0e7b19c9c7610dda","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115205598982180864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"closed","pr_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","final_merged_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T22:14:48Z","started_at":"2026-10-27T22:14:48Z","completed_at":"2026-10-27T22:32:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:46:14.155116542Z","timing_metadata_checked_at":"2026-10-27T22:46:14.155116542Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-27T22:46:14.160820427Z","region_metadata_checked_at":"2026-10-27T23:49:32.350433281Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"78d67969165767fb9596922cc1481fbf3c5aceb1da863d13dfa70c1ddedc2373","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115205598982180864","occurred_at":"2026-10-27T22:14:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115201309089992704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7544,"pr_state":"open","pr_sha":"ed7741c7e007b22fcf2a10919d0cf5a727c16488","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T21:57:45Z","started_at":"2026-10-27T21:57:45Z","completed_at":"2026-10-27T22:18:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:36:05.020578016Z","timing_metadata_checked_at":"2026-10-27T22:36:05.020578016Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-27T22:36:05.024584403Z","region_metadata_checked_at":"2026-10-27T23:39:02.07106068Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4f912118548119457ee950b05762b6a9518c43765bb1173a6f17c91c73ebedd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115201309089992704","occurred_at":"2026-10-27T21:57:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115193188430909440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7670,"pr_state":"open","pr_sha":"796b23786c72f3ccece6e5b6331bd272a6eccf3a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T21:25:29Z","started_at":"2026-10-27T21:25:29Z","completed_at":"2026-10-27T21:58:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:15:03.210118539Z","timing_metadata_checked_at":"2026-10-27T22:15:03.210118539Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T22:15:03.213469229Z","region_metadata_checked_at":"2026-10-27T22:15:03.213469229Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3417eed35036ed2c252664c7ae24781b7ccfbef0d590e67182cac532b5e49651","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115193188430909440","occurred_at":"2026-10-27T21:25:29Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T21:46:01.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T21:46:03.401Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T21:46:05.062Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ec85056f79198d17f9a4a06a9d62ec213e8ff2f13643b300963fd031a60589b0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0909440-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fec85056f79198d17f9a4a06a9d62ec213e8ff2f13643b300963fd031a60589b0\ntime=2026-10-27T21:48:05.587Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.Resources/deployments/ec85056f79198d17f9a4a06a9d62ec213e8ff2f13643b300963fd031a60589b0/operationStatuses/08584084693213550197\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.Resources/deployments/cluster-r5scxlmdedogk\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0909440-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j0909440-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-27T21:50:05.589Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T21:50:06.576Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T21:50:08.087Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=cd575f368230e7d22208e5f91e10168040367ee73eee75ae5c003a42111d906b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0909440-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd575f368230e7d22208e5f91e10168040367ee73eee75ae5c003a42111d906b\ntime=2026-10-27T21:51:38.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.Resources/deployments/cd575f368230e7d22208e5f91e10168040367ee73eee75ae5c003a42111d906b/operationStatuses/08584084690782406556\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7789/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115187930963120128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7789,"pr_state":"closed","pr_sha":"dea3567989f81bf1ac4109180a2a2ba486a9fc2d","final_merged_sha":"dea3567989f81bf1ac4109180a2a2ba486a9fc2d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T21:04:35Z","started_at":"2026-10-27T21:04:35Z","completed_at":"2026-10-27T23:25:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:39:02.269460754Z","timing_metadata_checked_at":"2026-10-27T23:39:02.269460754Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T23:39:02.275628435Z","region_metadata_checked_at":"2026-10-27T23:39:02.275628435Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115187836431896576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7548,"pr_state":"open","pr_sha":"41946db7801a7f0e0fd0bfb8a41eeaf6516f9f99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T21:04:13Z","started_at":"2026-10-27T21:04:13Z","completed_at":"2026-10-27T23:20:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:39:02.165949981Z","timing_metadata_checked_at":"2026-10-27T23:39:02.165949981Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T23:39:02.171156164Z","region_metadata_checked_at":"2026-10-27T23:39:02.171156164Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115181315694268416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7786,"pr_state":"closed","pr_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","final_merged_sha":"390355407ed3a643ff9b60a0e76f7b1769882838","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T20:38:18Z","started_at":"2026-10-27T20:38:18Z","completed_at":"2026-10-27T22:50:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:07:15.579794863Z","timing_metadata_checked_at":"2026-10-27T23:07:15.579794863Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T23:07:15.584371649Z","region_metadata_checked_at":"2026-10-27T23:07:15.584371649Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6aac1eac70425ca9e3a91aa0b05d3f05a56c627f8c79889631af5f48e74bcfbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115181315694268416","occurred_at":"2026-10-27T20:38:18Z","failed_at":"e2e","signature_id":"680a2853e0b95ed98a216673aae885a5a987ff0bbda27f4f85819f506da3a15f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27856e5eac20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xpvzdt8vb9wc\" to finish creating: GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"name\": \"1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:39:39.672372928Z\",\n      \"endTime\": \"2026-10-27T21:58:43.258733633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7h4b0s5o2n2n4p-np-1-29gbc-4c99s: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xpvzdt8vb9wc--managed/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xpvzdt8vb9wc--managed/providers/Microsoft.Network/networkInterfaces/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n            correlation ID: c4cb1e97-d5ea-4bde-b8e9-0601e9a9af7f\n            client request ID: b322bb9e-3c1b-45b3-9299-27b3632803d2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27856e5eac20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xpvzdt8vb9wc\" to finish creating: GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"name\": \"1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:39:39.672372928Z\",\n      \"endTime\": \"2026-10-27T21:58:43.258733633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7h4b0s5o2n2n4p-np-1-29gbc-4c99s: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xpvzdt8vb9wc--managed/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xpvzdt8vb9wc--managed/providers/Microsoft.Network/networkInterfaces/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n            correlation ID: c4cb1e97-d5ea-4bde-b8e9-0601e9a9af7f\n            client request ID: b322bb9e-3c1b-45b3-9299-27b3632803d2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0183c80982ac91e40ea21123d1f1a20353ca5fecf0db33e487b670cf6d4d4996","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7c66f0fe58a78c1398df2a693ef97607935e8d9b3c32c362b3e1f8fe8165f619","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115181315694268416","occurred_at":"2026-10-27T20:38:18Z","failed_at":"e2e","signature_id":"b6c5bc582704a39e441546c12f0a5da8432ee3cd1585874fdfcf70c165b8f04c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xbc827a50558\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xbc827a00340\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0xbc827b00060\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0xbc826b52230\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xbc827a50558\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xbc827a00340\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0xbc827b00060\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0xbc826b52230\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0183c80982ac91e40ea21123d1f1a20353ca5fecf0db33e487b670cf6d4d4996","0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115177325761925120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7667,"pr_state":"closed","pr_sha":"59ae68cb1d04712c5175790a72ad028c91c17203","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T20:22:27Z","started_at":"2026-10-27T20:22:27Z","completed_at":"2026-10-27T22:37:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:56:49.60223836Z","timing_metadata_checked_at":"2026-10-27T22:56:49.60223836Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T22:56:49.60541385Z","region_metadata_checked_at":"2026-10-27T22:56:49.60541385Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175058346348544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"closed","pr_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","final_merged_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T20:13:18Z","started_at":"2026-10-27T20:13:18Z","completed_at":"2026-10-27T21:17:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:33:03.646739844Z","timing_metadata_checked_at":"2026-10-27T21:33:03.646739844Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:33:03.650450632Z","region_metadata_checked_at":"2026-10-27T21:33:03.650450632Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4cb1087edc78a9ad7dc764a05c35df477454a6e56c7a0da98967a13ea68de3cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175058346348544","occurred_at":"2026-10-27T20:13:18Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T21:04:40.560Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-27T21:04:41.167Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-27T21:04:41.720Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=07716c56beddc7b7764d03db8be5e4423d70cd91036a0e2812ce90dae36950e0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6348544-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F07716c56beddc7b7764d03db8be5e4423d70cd91036a0e2812ce90dae36950e0\ntime=2026-10-27T21:12:43.185Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6348544-mgmt-1/providers/Microsoft.Resources/deployments/07716c56beddc7b7764d03db8be5e4423d70cd91036a0e2812ce90dae36950e0/operationStatuses/08584084718040326968\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6348544-mgmt-1/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/27/2026 9:05:02 PM' with correlationId '91b74ae1-3277-41f6-8cd7-59f2d0861c2c', and will expire at '11/3/2026 9:05:02 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c11ecd2e0a07f870d594200c753517b1b1da13ef27c39ff3dad4af7f1e59078f","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c11ecd2e0a07f870d594200c753517b1b1da13ef27c39ff3dad4af7f1e59078f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"63ee057fa8457e9b8d70fd6b234bbed2bb83e872","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T20:13:15Z","started_at":"2026-10-27T20:13:15Z","completed_at":"2026-10-27T22:40:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:56:49.76581577Z","timing_metadata_checked_at":"2026-10-27T22:56:49.76581577Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T22:56:49.768632061Z","region_metadata_checked_at":"2026-10-27T22:56:49.768632061Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1a8c66b96f7ea03e508fcab2aad8369747ccf89de1bb8406016ae2f709ea0957","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","failed_at":"alert","signature_id":"9ab9f28587da746e56d2ab1ae50303307906627a29629f6be6120283acdd04ee","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T21:38:22Z\n  Ended:       2026-10-27T22:04:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8626944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626944/providers/microsoft.monitor/accounts/hcps-j8626944\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9m2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T21:39:21Z\n  Ended:       2026-10-27T22:04:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8626944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626944/providers/microsoft.monitor/accounts/hcps-j8626944\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9m2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"201c1ded4a20b3ee028da2eb9a37e952459fd08873dfb0228f6a71ec05120797","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","failed_at":"alert","signature_id":"6651f88688a55750821d5c7f03935d0d70244cf6e6fd344d7bf85123f2fcc04f","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T21:41:15Z\n  Ended:       2026-10-27T22:09:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8626944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626944/providers/microsoft.monitor/accounts/hcps-j8626944\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f3fe228f-1ce0-4bce-a91a-29cdfd52c071\"\n  Description: Pod ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T21:41:15Z\n  Ended:       2026-10-27T22:09:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8626944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626944/providers/microsoft.monitor/accounts/hcps-j8626944\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f3fe228f-1ce0-4bce-a91a-29cdfd52c071\"\n  Description: Pod ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"851607a2a6a75f696eea945d8231ef94f49372b2ebb1e8137e6769734a9aa3a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","failed_at":"e2e","signature_id":"4a9d64725d8b6bbd0f0f8918c4d21324df99b2aeb2db108ba6a317b2bf3e34ad","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1620db330380\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gfmmkfqxc5x6\" to finish creating: GET https://rp.j8626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb0ecfe0-4f44-41f1-923e-4f3520584861\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb0ecfe0-4f44-41f1-923e-4f3520584861\",\n      \"name\": \"cb0ecfe0-4f44-41f1-923e-4f3520584861\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:18:30.659898808Z\",\n      \"endTime\": \"2026-10-27T21:47:37.431864682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb0ecfe0-4f44-41f1-923e-4f3520584861\n            correlation ID: 63848e13-133e-4fce-9179-580109e2b00e\n            client request ID: a25acb69-a9bf-4e2d-8af8-06447ee114b4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1620db330380\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gfmmkfqxc5x6\" to finish creating: GET https://rp.j8626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb0ecfe0-4f44-41f1-923e-4f3520584861\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb0ecfe0-4f44-41f1-923e-4f3520584861\",\n      \"name\": \"cb0ecfe0-4f44-41f1-923e-4f3520584861\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:18:30.659898808Z\",\n      \"endTime\": \"2026-10-27T21:47:37.431864682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb0ecfe0-4f44-41f1-923e-4f3520584861\n            correlation ID: 63848e13-133e-4fce-9179-580109e2b00e\n            client request ID: a25acb69-a9bf-4e2d-8af8-06447ee114b4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c39e8bf9fdfb2aba13c010bbc41c7656e40196d95ddd2a063b49e9b0569e4a41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","failed_at":"alert","signature_id":"d52355b973893083e742037f672c037fc0a285c782d00e94d81974354b675606","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T21:53:20Z\n  Ended:       2026-10-27T22:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8626944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626944/providers/microsoft.monitor/accounts/hcps-j8626944\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9m2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T21:53:20Z\n  Ended:       2026-10-27T22:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8626944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8626944/providers/microsoft.monitor/accounts/hcps-j8626944\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-r9m2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tnce9sp5mvj0fek8lfnjcm2ej2a4s3g-ea-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174942373842944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6081,"pr_state":"open","pr_sha":"860fe9f16ea44af6ed07ffbd4b0e9c062b01f3d7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T20:12:58Z","started_at":"2026-10-27T20:12:58Z","completed_at":"2026-10-27T22:37:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:56:49.284869711Z","timing_metadata_checked_at":"2026-10-27T22:56:49.284869711Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T22:56:49.288589099Z","region_metadata_checked_at":"2026-10-27T22:56:49.288589099Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1d5e66748ab1117fa02c23fede918e5fd5f02488bdd0d1730c61542084a3557c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174942373842944","occurred_at":"2026-10-27T20:12:58Z","failed_at":"alert","signature_id":"c48979391c5633bf7cf1f49fba3381dc030e10cbe77d463ae9e3618fb8f7821a","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T21:37:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j3842944-svc\", severity=\"warning\"\n  Description: Credential refresher expiration metric is missing for service cluster ci01-j3842944-svc.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583","failure_pattern":"alert [svc] ClusterCredentialMetricMissing fired","search_query":"alert [svc] ClusterCredentialMetricMissing fired"}},{"row_id":"93a900c23656e86ce0e1f46edb30728d2380533ddf11641d48a68dcecba5b2e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174942373842944","occurred_at":"2026-10-27T20:12:58Z","failed_at":"e2e","signature_id":"ac88b44d49d4b3fc31eb516f5c18cc6ed9c7017beeca05d29331dc6a4f6b0be6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dj4ptk\"\nUnexpected error:\n    \u003c*errors.joinError | 0xd5290aa85b8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xd5290a963a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                err: \u003c*fmt.wrapError | 0xd5290356ce0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                    err: \u003c*verifiers.pollObservationError | 0xd5290356540\u003e{\n                        observation: \u003c*errors.errorString | 0xd52904125d0\u003e{\n                            s: \"PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0xd5290356520\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-8ql64/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dj4ptk\"\nUnexpected error:\n    \u003c*errors.joinError | 0xd5290aa85b8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xd5290a963a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                err: \u003c*fmt.wrapError | 0xd5290356ce0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                    err: \u003c*verifiers.pollObservationError | 0xd5290356540\u003e{\n                        observation: \u003c*errors.errorString | 0xd52904125d0\u003e{\n                            s: \"PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0xd5290356520\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-8ql64/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"266fecc1d3355c9965ab1cf579cbe09c8cc3762b85e6438d6ae6f2889f5af561","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)","search_query":"[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["266fecc1d3355c9965ab1cf579cbe09c8cc3762b85e6438d6ae6f2889f5af561","e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174872618373120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7769,"pr_state":"closed","pr_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","final_merged_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T20:12:42Z","started_at":"2026-10-27T20:12:42Z","completed_at":"2026-10-27T22:55:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:07:15.475102777Z","timing_metadata_checked_at":"2026-10-27T23:07:15.475102777Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T23:07:15.479868563Z","region_metadata_checked_at":"2026-10-27T23:07:15.479868563Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a2d3407d24536e85f11b193aa2fced2ea9d251210c9c702adc4f95e1dbf08bc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174872618373120","occurred_at":"2026-10-27T20:12:42Z","failed_at":"e2e","signature_id":"2bba532e021c86d2ed01a12b037a58a40fc2823f364697d5f4bcfc466e421a66","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-rvmpld\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aa0726d2620\u003e: \n    DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aa071c17780\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-rvmpld\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aa0726d2620\u003e: \n    DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aa071c17780\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115165748904071168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"d277c61d9afc52f462ec7c99644c51c7773d1ee1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T19:36:27Z","started_at":"2026-10-27T19:36:27Z","completed_at":"2026-10-27T22:50:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T23:07:15.198727607Z","timing_metadata_checked_at":"2026-10-27T23:07:15.198727607Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T23:07:15.202281096Z","region_metadata_checked_at":"2026-10-27T23:07:15.202281096Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fa071cbf03b60b1cbdb5fa0483126c83feb606702bf47eaa187e29c8405dd999","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115165748904071168","occurred_at":"2026-10-27T19:36:27Z","failed_at":"e2e","signature_id":"5b6c70c60af9afcab97207b8cdf90bb3550c811d6b46f808c94720b90fbcfe2e","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cc8b3a2b3a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1cc8b3559460\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cc8b3a2b3a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1cc8b3559460\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115164743516819456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7739,"pr_state":"closed","pr_sha":"8a870f830cea08dda15ce26798b6c9f935c71173","final_merged_sha":"85f478eff30207b95d439e1d26791f75c7ff20f7","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T19:32:25Z","started_at":"2026-10-27T19:32:25Z","completed_at":"2026-10-27T21:52:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:04:29.125412516Z","timing_metadata_checked_at":"2026-10-27T22:04:29.125412516Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T22:04:29.128670306Z","region_metadata_checked_at":"2026-10-27T22:04:29.128670306Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115163835793936384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5244,"pr_state":"open","pr_sha":"d4550ee62563a67a9b6ad205492381d75b0be718","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T19:28:50Z","started_at":"2026-10-27T19:28:50Z","completed_at":"2026-10-27T21:57:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T22:15:03.085581608Z","timing_metadata_checked_at":"2026-10-27T22:15:03.085581608Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T22:15:03.089110098Z","region_metadata_checked_at":"2026-10-27T22:15:03.089110098Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1dcb047027e531e840f279bc6248b86ae6e4d42accd510f08f06a3c77ab8aec0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115163835793936384","occurred_at":"2026-10-27T19:28:50Z","failed_at":"e2e","signature_id":"d07a57d85a5fbf19cecde1119f1ea8aa1c9e7ad4e3e554a72b71831011228ce2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:545]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x303527456700\u003e: \n    DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3035274566c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:545]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x303527456700\u003e: \n    DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3035274566c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-cv7tq"}},{"row_id":"ed7ce4a616ea25e18faf6a83adde6a69b236964b519060e9fe5249722e7ebac6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115163835793936384","occurred_at":"2026-10-27T19:28:50Z","failed_at":"e2e","signature_id":"6933fc9bb0db831be293275832c4bad7fce83abb554cf2ccb7be65585cfd3714","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x267931bca260\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hpqngqxblkt5\" to finish creating: GET https://rp.j3936384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ce6c32c-2374-4848-87d4-d06da5841ac0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ce6c32c-2374-4848-87d4-d06da5841ac0\",\n      \"name\": \"4ce6c32c-2374-4848-87d4-d06da5841ac0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T20:35:56.651068091Z\",\n      \"endTime\": \"2026-10-27T20:55:03.998387798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-npwtg-fbvfl: Failed: virtualmachine failed to create or update. err: failed to get existing resource v20260901preview-hpqngqxblkt5--managed/v20260901-np-1-npwtg-fbvfl (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hpqngqxblkt5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-npwtg-fbvfl\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3936384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ce6c32c-2374-4848-87d4-d06da5841ac0\n            correlation ID: d0468374-ad1f-4a29-ba7f-fc2cf91bd94a\n            client request ID: 907abad4-e4b1-4719-9ea0-4da30fbe13ee\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x267931bca260\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hpqngqxblkt5\" to finish creating: GET https://rp.j3936384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ce6c32c-2374-4848-87d4-d06da5841ac0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ce6c32c-2374-4848-87d4-d06da5841ac0\",\n      \"name\": \"4ce6c32c-2374-4848-87d4-d06da5841ac0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T20:35:56.651068091Z\",\n      \"endTime\": \"2026-10-27T20:55:03.998387798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-npwtg-fbvfl: Failed: virtualmachine failed to create or update. err: failed to get existing resource v20260901preview-hpqngqxblkt5--managed/v20260901-np-1-npwtg-fbvfl (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hpqngqxblkt5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-npwtg-fbvfl\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3936384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ce6c32c-2374-4848-87d4-d06da5841ac0\n            correlation ID: d0468374-ad1f-4a29-ba7f-fc2cf91bd94a\n            client request ID: 907abad4-e4b1-4719-9ea0-4da30fbe13ee\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115159627384492032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"closed","pr_sha":"c8ebae4103c841cfb1590b88cc329be1a9829823","final_merged_sha":"c8ebae4103c841cfb1590b88cc329be1a9829823","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T19:10:54Z","started_at":"2026-10-27T19:10:54Z","completed_at":"2026-10-27T21:24:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:33:03.756660109Z","timing_metadata_checked_at":"2026-10-27T21:33:03.756660109Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:33:03.760295098Z","region_metadata_checked_at":"2026-10-27T21:33:03.760295098Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115161245337260032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7700,"pr_state":"closed","pr_sha":"50bf39bf591aa2b64e5bedba4ba2f4443d76d279","final_merged_sha":"50bf39bf591aa2b64e5bedba4ba2f4443d76d279","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T19:07:22Z","started_at":"2026-10-27T19:07:22Z","completed_at":"2026-10-27T21:26:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:43:32.129586355Z","timing_metadata_checked_at":"2026-10-27T21:43:32.129586355Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:43:32.134825539Z","region_metadata_checked_at":"2026-10-27T21:43:32.134825539Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7436/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115159601987981312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7436,"pr_state":"open","pr_sha":"8759c9fb5b7d83158666e5614c13d6c092220390","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T18:59:37Z","started_at":"2026-10-27T18:59:37Z","completed_at":"2026-10-27T21:20:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:33:02.394478257Z","timing_metadata_checked_at":"2026-10-27T21:33:02.394478257Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:33:02.398194246Z","region_metadata_checked_at":"2026-10-27T21:33:02.398194246Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"89226263e6ab928b6cab595e90173662d438585af6b5a540aa3f1e3970731e4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7436/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115159601987981312","occurred_at":"2026-10-27T18:59:37Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T20:20:10Z\n  Ended:       2026-10-27T20:29:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7981312-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115158357760282624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T18:58:44Z","started_at":"2026-10-27T18:58:44Z","completed_at":"2026-10-27T21:17:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:33:03.895534386Z","timing_metadata_checked_at":"2026-10-27T21:33:03.895534386Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:33:03.899202775Z","region_metadata_checked_at":"2026-10-27T21:33:03.899202775Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7784/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115152661329219585","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7784,"pr_state":"closed","pr_sha":"2492d65ed9f62aacb5f9025492239c8ffb7547e3","final_merged_sha":"2492d65ed9f62aacb5f9025492239c8ffb7547e3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T18:41:27Z","started_at":"2026-10-27T18:41:27Z","completed_at":"2026-10-27T21:07:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:22:04.348467439Z","timing_metadata_checked_at":"2026-10-27T21:22:04.348467439Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:22:04.351798629Z","region_metadata_checked_at":"2026-10-27T21:22:04.351798629Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115149099081666560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7781,"pr_state":"open","pr_sha":"190ff050d8cfd8c36061832861f8119ff502f262","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T17:40:53Z","started_at":"2026-10-27T17:40:53Z","completed_at":"2026-10-27T21:05:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T21:22:03.93547212Z","timing_metadata_checked_at":"2026-10-27T21:22:03.93547212Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T21:22:03.939096608Z","region_metadata_checked_at":"2026-10-27T21:22:03.939096608Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b0f2d1f0b503dfdd462cc1a542d922a43f6ea5949fd49f1773962614ecd2231d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115149099081666560","occurred_at":"2026-10-27T17:40:53Z","failed_at":"e2e","signature_id":"59414b931b98b1d5d286decc70f822294d05cabdb5a1afce423cbe142550f9fb","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c3a9a99f660\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-hqzrkb5qdbkx\" to finish creating: GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"name\": \"ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T20:11:38.810985949Z\",\n      \"endTime\": \"2026-10-27T20:30:42.147810175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n            correlation ID: 20b2a10f-dd2b-4a62-8cff-21fa59649da0\n            client request ID: a877eec2-04c5-44bd-9434-61383daeb4f5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c3a9a99f660\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-hqzrkb5qdbkx\" to finish creating: GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"name\": \"ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T20:11:38.810985949Z\",\n      \"endTime\": \"2026-10-27T20:30:42.147810175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n            correlation ID: 20b2a10f-dd2b-4a62-8cff-21fa59649da0\n            client request ID: a877eec2-04c5-44bd-9434-61383daeb4f5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115141408217108480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7780,"pr_state":"open","pr_sha":"0701c5f57513cf9e3affdd6a6efcb9804da452d2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T17:26:21Z","started_at":"2026-10-27T17:26:21Z","completed_at":"2026-10-27T20:04:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T20:19:07.905877023Z","timing_metadata_checked_at":"2026-10-27T20:19:07.905877023Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T20:19:07.908952013Z","region_metadata_checked_at":"2026-10-27T20:19:07.908952013Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115143321788616704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"closed","pr_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","final_merged_sha":"cd3c21e53a196ba0ad2c38b6f73f33b744c9e3fa","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T17:25:32Z","started_at":"2026-10-27T17:25:32Z","completed_at":"2026-10-27T20:19:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T20:29:55.348145862Z","timing_metadata_checked_at":"2026-10-27T20:29:55.348145862Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T20:29:55.351499652Z","region_metadata_checked_at":"2026-10-27T20:29:55.351499652Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7753/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127534017318912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7753,"pr_state":"closed","pr_sha":"659a47e6b05209a86c11c1924a1b72a01a79d4a0","final_merged_sha":"659a47e6b05209a86c11c1924a1b72a01a79d4a0","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T17:04:34Z","started_at":"2026-10-27T17:04:34Z","completed_at":"2026-10-27T18:13:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:23:37.041128577Z","timing_metadata_checked_at":"2026-10-27T18:23:37.041128577Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:23:37.049805249Z","region_metadata_checked_at":"2026-10-27T18:23:37.049805249Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"87cef38f143ccdd0b66a0288b9b6cc25725f09f597adc376e0f32d2c23253c94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7753/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127534017318912","occurred_at":"2026-10-27T17:04:34Z","failed_at":"provision","signature_id":"4fbcbcae7d0f352f867128831decf8a2cd0c0541228c3392fa864aa36c4d6550","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T17:52:24.476Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-10-27T17:52:24.848Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-10-27T17:52:24.848Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-27T17:52:24.880Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 namespace=multicluster-engine\ntime=2026-10-27T17:52:24.933Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 dryRun=false\ntime=2026-10-27T17:52:24.933Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-27T17:52:24.990Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-27T18:02:49.740Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-27T18:02:49.741Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127036694499328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6081,"pr_state":"open","pr_sha":"860fe9f16ea44af6ed07ffbd4b0e9c062b01f3d7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T17:02:37Z","started_at":"2026-10-27T17:02:37Z","completed_at":"2026-10-27T19:22:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T19:37:15.342439322Z","timing_metadata_checked_at":"2026-10-27T19:37:15.342439322Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T19:37:15.346241612Z","region_metadata_checked_at":"2026-10-27T19:37:15.346241612Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"589ca8b7c6c4e54f58829f63684e2412e9e90831198a3d1b659dd53f44ea6344","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127036694499328","occurred_at":"2026-10-27T17:02:37Z","failed_at":"e2e","signature_id":"d64f17ee0348ee472ddc21b645d0031133903222e72b9d637406698ac14c2738","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-6lzc2b\"\nUnexpected error:\n    \u003c*errors.joinError | 0x263fb059e4c8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x263fb0784660\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                err: \u003c*fmt.wrapError | 0x263fb09fc440\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                    err: \u003c*errors.errorString | 0x263fb0ff0b50\u003e{\n                        s: \"PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-6lzc2b\"\nUnexpected error:\n    \u003c*errors.joinError | 0x263fb059e4c8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x263fb0784660\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                err: \u003c*fmt.wrapError | 0x263fb09fc440\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                    err: \u003c*errors.errorString | 0x263fb0ff0b50\u003e{\n                        s: \"PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ba04fee393b18d4d6b04ac1f68b72a6345177ebad95a48fd67494f2e5f295e14","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)","search_query":"[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)"}},{"row_id":"b15bf11df8d3530e43197788b8b525a0f4672f3934de190f2a157a07aa072abe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127036694499328","occurred_at":"2026-10-27T17:02:37Z","failed_at":"alert","signature_id":"bdaf4cc6230c0864f55513a6432317c85b8bb17698744582bcc8c62c19c0698b","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T18:18:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j4499328-svc\", severity=\"warning\"\n  Description: Credential refresher expiration metric is missing for service cluster ci01-j4499328-svc.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583","failure_pattern":"alert [svc] ClusterCredentialMetricMissing fired","search_query":"alert [svc] ClusterCredentialMetricMissing fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["ba04fee393b18d4d6b04ac1f68b72a6345177ebad95a48fd67494f2e5f295e14","e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115122591927635968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T16:44:57Z","started_at":"2026-10-27T16:44:57Z","completed_at":"2026-10-27T18:58:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T19:15:25.447702939Z","timing_metadata_checked_at":"2026-10-27T19:15:25.447702939Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T19:15:25.451370927Z","region_metadata_checked_at":"2026-10-27T19:15:25.451370927Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"279e48aac226650f4a9206d91fc932679fb7ecfa804affe930b4c42fb0d99df7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115122591927635968","occurred_at":"2026-10-27T16:44:57Z","failed_at":"e2e","signature_id":"565fcf6646d614a2f928329c9d0fc3e5adde16bae5d4e208002a59dbb76d5fe4","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1218a86ea3a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-trmq28plbffk\" to finish creating: GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\",\n      \"name\": \"4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:48:09.944151958Z\",\n      \"endTime\": \"2026-10-27T18:07:12.889340309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-vq2bz-jbwhf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-vq2bz-jbwhf: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-trmq28plbffk--managed/ingress-cert-le-np-1-vq2bz-jbwhf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-trmq28plbffk--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-vq2bz-jbwhf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\n            correlation ID: 08a9a0eb-cef3-4f31-9ee9-fe86da6b3d5c\n            client request ID: 4911c042-45cb-4cb5-8e27-b7f9d98a083a\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1218a86ea3a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-trmq28plbffk\" to finish creating: GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\",\n      \"name\": \"4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:48:09.944151958Z\",\n      \"endTime\": \"2026-10-27T18:07:12.889340309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-vq2bz-jbwhf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-vq2bz-jbwhf: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-trmq28plbffk--managed/ingress-cert-le-np-1-vq2bz-jbwhf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-trmq28plbffk--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-vq2bz-jbwhf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dccef60-3d8d-4da6-abc7-6f50fc2d7fea\n            correlation ID: 08a9a0eb-cef3-4f31-9ee9-fe86da6b3d5c\n            client request ID: 4911c042-45cb-4cb5-8e27-b7f9d98a083a","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp"}},{"row_id":"943aec8c0191cb1c656bb7e59bdbc48f1b24b31750da75877246352ff767d39d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115122591927635968","occurred_at":"2026-10-27T16:44:57Z","failed_at":"e2e","signature_id":"cdf4ff3359680941ca4519be04a9ab48255bff7cfae013bdf33380df365386e8","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d4aea46a060\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-85bttqbvwzdg\" to finish creating: GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"name\": \"7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:47:01.433391882Z\",\n      \"endTime\": \"2026-10-27T18:06:02.631484435Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n            correlation ID: b503cd11-01c7-4b49-bd8f-b11bdfa64e3b\n            client request ID: 8e981bdf-718f-4374-bc9d-9fe858083611\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d4aea46a060\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-85bttqbvwzdg\" to finish creating: GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"name\": \"7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:47:01.433391882Z\",\n      \"endTime\": \"2026-10-27T18:06:02.631484435Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n            correlation ID: b503cd11-01c7-4b49-bd8f-b11bdfa64e3b\n            client request ID: 8e981bdf-718f-4374-bc9d-9fe858083611\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115122020378218496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"a630f5d29ba754ca53a2c8994a802571baafa9d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T16:42:41Z","started_at":"2026-10-27T16:42:41Z","completed_at":"2026-10-27T18:59:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T19:15:25.310966477Z","timing_metadata_checked_at":"2026-10-27T19:15:25.310966477Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T19:15:25.314578965Z","region_metadata_checked_at":"2026-10-27T19:15:25.314578965Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115121597042921472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7770,"pr_state":"closed","pr_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","final_merged_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T16:41:00Z","started_at":"2026-10-27T16:41:00Z","completed_at":"2026-10-27T18:59:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T19:15:25.579190318Z","timing_metadata_checked_at":"2026-10-27T19:15:25.579190318Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T19:15:25.582188209Z","region_metadata_checked_at":"2026-10-27T19:15:25.582188209Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115118404208693248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"closed","pr_sha":"c8ebae4103c841cfb1590b88cc329be1a9829823","final_merged_sha":"c8ebae4103c841cfb1590b88cc329be1a9829823","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T16:28:19Z","started_at":"2026-10-27T16:28:19Z","completed_at":"2026-10-27T18:45:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:54:17.301981186Z","timing_metadata_checked_at":"2026-10-27T18:54:17.301981186Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:54:17.305441375Z","region_metadata_checked_at":"2026-10-27T18:54:17.305441375Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4877b44df5c0623c4811b15cba7bcffe0a12776a4bbeff2f062be470f2948563","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115118404208693248","occurred_at":"2026-10-27T16:28:19Z","failed_at":"e2e","signature_id":"adcabdd64318a6bbc42056fc4a3695e48135925c38c3cccf257c171d24c1fa4c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-20-csbqrn\"\nUnexpected error:\n    \u003c*errors.joinError | 0x395999ae65e8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x39599a424e20\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0x395999e53a40\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0x39599931d360\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-20-csbqrn\"\nUnexpected error:\n    \u003c*errors.joinError | 0x395999ae65e8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x39599a424e20\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0x395999e53a40\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0x39599931d360\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8d06c33c9ca8da47b85ff204ce768771e156eaca59fdc3f367974352e4e17539","failure_pattern":"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod \u003cnamespace\u003e/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pod \u003cpod\u003e)","search_query":"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8d06c33c9ca8da47b85ff204ce768771e156eaca59fdc3f367974352e4e17539"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115114555087523840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T16:12:49Z","started_at":"2026-10-27T16:12:49Z","completed_at":"2026-10-27T20:05:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T20:19:07.68293673Z","timing_metadata_checked_at":"2026-10-27T20:19:07.68293673Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T20:19:07.6924186Z","region_metadata_checked_at":"2026-10-27T20:19:07.6924186Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112713075036160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T16:05:42Z","started_at":"2026-10-27T16:05:42Z","completed_at":"2026-10-27T16:40:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.204080245Z","timing_metadata_checked_at":"2026-10-27T16:50:27.204080245Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.206859336Z","region_metadata_checked_at":"2026-10-27T16:50:27.206859336Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ed335da809902dad6e290dd059868736599945eab470e0636f9a9540db4706f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112713075036160","occurred_at":"2026-10-27T16:05:42Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T16:22:47.857Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T16:22:49.418Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T16:22:50.992Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=7ad6b585c2a0db4b1394b6a9516e5015c1679a296a1e0d17bcbd7a3fba0e88c5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5036160-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7ad6b585c2a0db4b1394b6a9516e5015c1679a296a1e0d17bcbd7a3fba0e88c5\ntime=2026-10-27T16:25:21.585Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5036160-svc/providers/Microsoft.Resources/deployments/7ad6b585c2a0db4b1394b6a9516e5015c1679a296a1e0d17bcbd7a3fba0e88c5/operationStatuses/08584084887153570073\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5036160-svc/providers/Microsoft.Resources/deployments/cluster-iq4rpd4u335k2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5036160-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5036160-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j5036160-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-27T16:27:21.586Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T16:27:22.568Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T16:27:27.384Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=3dc95d5c58fec68b29e80b8f5b89cd82220ecff7029d45288f7b18d7b85bc595 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5036160-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3dc95d5c58fec68b29e80b8f5b89cd82220ecff7029d45288f7b18d7b85bc595\ntime=2026-10-27T16:28:57.922Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5036160-svc/providers/Microsoft.Resources/deployments/3dc95d5c58fec68b29e80b8f5b89cd82220ecff7029d45288f7b18d7b85bc595/operationStatuses/08584084884394674722\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112256785092608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"2b7d53ac2b56f8cf2ef6020b592169ab9f88b624","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T16:03:40Z","started_at":"2026-10-27T16:03:40Z","completed_at":"2026-10-27T18:19:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:33:39.984375391Z","timing_metadata_checked_at":"2026-10-27T18:33:39.984375391Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:33:39.995739556Z","region_metadata_checked_at":"2026-10-27T18:33:39.995739556Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"17f211be08fd880904d6b76ae51794d45adc8777ecf6fde72d2a4c674b5ce9f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112256785092608","occurred_at":"2026-10-27T16:03:40Z","failed_at":"e2e","signature_id":"c6ac03d3d873d7b532f5a92eba3e5d864ae3c62d4f24763727d7aa7fc6ab9621","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.007s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\nUnexpected error:\n    \u003c*url.Error | 0xc2056a72b10\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\",\n        Err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.007s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\nUnexpected error:\n    \u003c*url.Error | 0xc2056a72b10\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\",\n        Err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"73cc879e56f570416795adcce7e08e28387abc22a0432f0cca452338bd4aff84","failure_pattern":"Get \"\u003curl\u003e\": context deadline exceeded","search_query":"Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded"}},{"row_id":"504024e0206a176bb4613f943796acc6256248750ae40272254c4585a410341b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112256785092608","occurred_at":"2026-10-27T16:03:40Z","failed_at":"e2e","signature_id":"f9f25242b172db559f6d87423a34b4f0d1ff324276218bc210554fe6763ef622","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x141a49f31000\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-czkfqz2c2dgm\" to finish creating: GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"name\": \"8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:03:30.992910534Z\",\n      \"endTime\": \"2026-10-27T17:22:33.122027279Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllMachinesReady is False: 2 of 4 machines are not ready\\nMachine u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-czkfqz2c2dgm--managed/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-czkfqz2c2dgm--managed/providers/Microsoft.Network/networkInterfaces/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n            correlation ID: 7178816c-96c8-477b-bc2c-2cc5e707ac42\n            client request ID: 3fb94b9b-1ccb-4142-9d30-c007dbf5afcb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x141a49f31000\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-czkfqz2c2dgm\" to finish creating: GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"name\": \"8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:03:30.992910534Z\",\n      \"endTime\": \"2026-10-27T17:22:33.122027279Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllMachinesReady is False: 2 of 4 machines are not ready\\nMachine u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-czkfqz2c2dgm--managed/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-czkfqz2c2dgm--managed/providers/Microsoft.Network/networkInterfaces/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n            correlation ID: 7178816c-96c8-477b-bc2c-2cc5e707ac42\n            client request ID: 3fb94b9b-1ccb-4142-9d30-c007dbf5afcb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55f249c389c8381077274234c503c49f7f7b8734eaf356caba055061beb868f8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["55f249c389c8381077274234c503c49f7f7b8734eaf356caba055061beb868f8","73cc879e56f570416795adcce7e08e28387abc22a0432f0cca452338bd4aff84"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115107245623611392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7311,"pr_state":"open","pr_sha":"eaa0e2dc204192d36b7631a36a3fbba446c8a24d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T15:43:58Z","started_at":"2026-10-27T15:43:58Z","completed_at":"2026-10-27T18:22:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:33:39.576483456Z","timing_metadata_checked_at":"2026-10-27T18:33:39.576483456Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:33:39.580007345Z","region_metadata_checked_at":"2026-10-27T18:33:39.580007345Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115105471453990912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"2d7c0c2df8df6e33733ba3d45cd981dec1e340e1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T15:36:47Z","started_at":"2026-10-27T15:36:47Z","completed_at":"2026-10-27T18:00:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:13:03.666977808Z","timing_metadata_checked_at":"2026-10-27T18:13:03.666977808Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:13:03.670085798Z","region_metadata_checked_at":"2026-10-27T18:13:03.670085798Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115103272862748672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T15:28:11Z","started_at":"2026-10-27T15:28:11Z","completed_at":"2026-10-27T15:47:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T15:57:05.245791402Z","timing_metadata_checked_at":"2026-10-27T15:57:05.245791402Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-27T15:57:05.249070491Z","region_metadata_checked_at":"2026-10-27T17:00:51.845308333Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"33dbaba2e9c4d55a80187366f8d90712c35fca1760b4a1fd868fc2155cfded97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115103272862748672","occurred_at":"2026-10-27T15:28:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115098874459721728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7700,"pr_state":"closed","pr_sha":"50bf39bf591aa2b64e5bedba4ba2f4443d76d279","final_merged_sha":"50bf39bf591aa2b64e5bedba4ba2f4443d76d279","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T15:10:42Z","started_at":"2026-10-27T15:10:42Z","completed_at":"2026-10-27T17:15:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T17:32:29.017696574Z","timing_metadata_checked_at":"2026-10-27T17:32:29.017696574Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T17:32:29.021279064Z","region_metadata_checked_at":"2026-10-27T17:32:29.021279064Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"09eda98d806a9a9bcdb83d81d1bf3d86920e563b0f3110fd72a7b4fa1206e093","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115098874459721728","occurred_at":"2026-10-27T15:10:42Z","failed_at":"e2e","signature_id":"29d10a662a78238c732f153094e693885b24ff3acbe770818665b4faa0fcd90a","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19ee49ef83c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wwr8wdxmg7md\" to finish creating: GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"name\": \"e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T16:09:15.328141673Z\",\n      \"endTime\": \"2026-10-27T16:28:23.766899649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n            correlation ID: 963eb402-1e02-4815-b759-a4a687cc24af\n            client request ID: 393f46a9-acc4-4c32-9f2a-527f2574f38e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19ee49ef83c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wwr8wdxmg7md\" to finish creating: GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"name\": \"e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T16:09:15.328141673Z\",\n      \"endTime\": \"2026-10-27T16:28:23.766899649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n            correlation ID: 963eb402-1e02-4815-b759-a4a687cc24af\n            client request ID: 393f46a9-acc4-4c32-9f2a-527f2574f38e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115097821592948736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6954,"pr_state":"open","pr_sha":"74ba89b62c4fa76ff63bd55c765c5112bfd64e90","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T15:06:31Z","started_at":"2026-10-27T15:06:31Z","completed_at":"2026-10-27T16:40:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.042110265Z","timing_metadata_checked_at":"2026-10-27T16:50:27.042110265Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.045366554Z","region_metadata_checked_at":"2026-10-27T16:50:27.045366554Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9beb3d75d401f1dea39822cff8d475d1ff2e0086b5edbd19341c1ae3e09d1ee2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115097821592948736","occurred_at":"2026-10-27T15:06:31Z","failed_at":"e2e","signature_id":"7e1c250216db8719c65e193ed331feca05376cad3f3ee5d0c571c8845c69f894","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:159]: cluster did not become visible in console.redhat.com\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f4ce1090c80\u003e: \n    VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\": can't get access token: invalid_grant: Invalid refresh token\n    {\n        msg: \"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n        err: \u003c*fmt.wrapError | 0x1f4ce110ccc0\u003e{\n            msg: \"failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n            err: \u003c*url.Error | 0x1f4ce0ce8a80\u003e{\n                Op: \"Get\",\n                URL: \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\",\n                Err: \u003c*fmt.wrapError | 0x1f4ce110cca0\u003e{\n                    msg: \"can't get access token: invalid_grant: Invalid refresh token\",\n                    err: \u003c*errors.errorString | 0x1f4ce1105790\u003e{\n                        s: \"invalid_grant: Invalid refresh token\",\n                    },\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:159]: cluster did not become visible in console.redhat.com\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f4ce1090c80\u003e: \n    VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\": can't get access token: invalid_grant: Invalid refresh token\n    {\n        msg: \"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n        err: \u003c*fmt.wrapError | 0x1f4ce110ccc0\u003e{\n            msg: \"failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n            err: \u003c*url.Error | 0x1f4ce0ce8a80\u003e{\n                Op: \"Get\",\n                URL: \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\",\n                Err: \u003c*fmt.wrapError | 0x1f4ce110cca0\u003e{\n                    msg: \"can't get access token: invalid_grant: Invalid refresh token\",\n                    err: \u003c*errors.errorString | 0x1f4ce1105790\u003e{\n                        s: \"invalid_grant: Invalid refresh token\",\n                    },\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aee4a820e8bbfea9c5305e481cde65284da0e2c801d92f6a858dc46e820847ed","failure_pattern":"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"\u003cuuid\u003e\": Get \"\u003curl\u003e\": can't get access token: invalid_grant: Invalid refresh token","search_query":"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\": can't get access token: invalid_grant: Invalid refresh token"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aee4a820e8bbfea9c5305e481cde65284da0e2c801d92f6a858dc46e820847ed"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7755/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115096588681482240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7755,"pr_state":"closed","pr_sha":"a1e0f6f839aaa629e285b4f0a328e73b7e407bea","final_merged_sha":"a1e0f6f839aaa629e285b4f0a328e73b7e407bea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T15:01:21Z","started_at":"2026-10-27T15:01:21Z","completed_at":"2026-10-27T17:18:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T17:32:29.39901021Z","timing_metadata_checked_at":"2026-10-27T17:32:29.39901021Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T17:32:29.402193801Z","region_metadata_checked_at":"2026-10-27T17:32:29.402193801Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115096568548823040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"closed","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T15:01:10Z","started_at":"2026-10-27T15:01:10Z","completed_at":"2026-10-27T18:18:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:33:39.777967531Z","timing_metadata_checked_at":"2026-10-27T18:33:39.777967531Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:33:39.78171152Z","region_metadata_checked_at":"2026-10-27T18:33:39.78171152Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7725/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115095913973157888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7725,"pr_state":"closed","pr_sha":"ddcdde4e996a0a299f14edd71f3fe9cfa025ce4a","final_merged_sha":"ddcdde4e996a0a299f14edd71f3fe9cfa025ce4a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T14:58:57Z","started_at":"2026-10-27T14:58:57Z","completed_at":"2026-10-27T17:16:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T17:32:29.209792138Z","timing_metadata_checked_at":"2026-10-27T17:32:29.209792138Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T17:32:29.213437028Z","region_metadata_checked_at":"2026-10-27T17:32:29.213437028Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115093743005601792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7670,"pr_state":"open","pr_sha":"d664c0dce9a3f2c73f8e0f0f7de4aae354f244d2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T14:50:19Z","started_at":"2026-10-27T14:50:19Z","completed_at":"2026-10-27T16:07:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:18:01.110555941Z","timing_metadata_checked_at":"2026-10-27T16:18:01.110555941Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:18:01.11390213Z","region_metadata_checked_at":"2026-10-27T16:18:01.11390213Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"50a1d57ffdd9078fe7b7989d21e9b9b18bd8c657e2ad4bfb1915f9c2afe2a733","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115093743005601792","occurred_at":"2026-10-27T14:50:19Z","failed_at":"provision","signature_id":"8aa20315f26c0b32e08e3b819a65f71c1dceb69f32d20ac323dffb320bb45d77","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T15:30:56.670Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-27T16:00:56.129Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-27T15:30:56.681953143Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.731109745Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.873423526Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a801ead30ba9f1a519d4f6e8321847a7265bbd197dc70d8fa10568c3510a622a","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5601792-mgmt-1\",\"clusterName\":\"ci01-j560179","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-27T15:30:56.681953143Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.731109745Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.873423526Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}},{"row_id":"9030898a4a20ebd8d224d0fc72298a5ff97e76bf65f3fb4efafbd0ea1157df4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115093743005601792","occurred_at":"2026-10-27T14:50:19Z","failed_at":"alert","signature_id":"9f8170d470887f01d9dcbfc134faa8684787533bfd7b84d71b542971a3f5c05d","test_name":"[aro-hcp-observability] [svc] alert userJourneyBackupMetricsAbsent does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:59:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyBackupMetricsAbsent\", cluster=\"ci01-j5601792-mgmt-2\", component=\"slo\", severity=\"3\", slo=\"backup-metrics-absent\"\n  Description: Management cluster ci01-j5601792-mgmt-2 is in the underlay-cluster inventory but has had no successful Velero server metrics scrape for 15m. Check Velero, its PodMonitor, and the management-cluster metrics pipeline. A healthy empty cluster with up=1 is not a failure merely because it has no backup attempts.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"73605e60345d9768cbfbea26978720be4c3942667f138969f6f3eb62f002e92e","failure_pattern":"alert [svc] userJourneyBackupMetricsAbsent fired","search_query":"alert [svc] userJourneyBackupMetricsAbsent fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["73605e60345d9768cbfbea26978720be4c3942667f138969f6f3eb62f002e92e","a801ead30ba9f1a519d4f6e8321847a7265bbd197dc70d8fa10568c3510a622a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115092150784888832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6718,"pr_state":"closed","pr_sha":"650947edd467aacb1759d718c9f05b200e6d2bc1","final_merged_sha":"de22b69e0ee31f17c2b9e52b1813cdd58df0d48f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T14:43:59Z","started_at":"2026-10-27T14:43:59Z","completed_at":"2026-10-27T17:55:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T18:13:03.468537326Z","timing_metadata_checked_at":"2026-10-27T18:13:03.468537326Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T18:13:03.472080715Z","region_metadata_checked_at":"2026-10-27T18:13:03.472080715Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f7a483d370b670d37faf25c359826afea2a49059cbe24c91749bf73db183892c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115092150784888832","occurred_at":"2026-10-27T14:43:59Z","failed_at":"e2e","signature_id":"052f4a60cc44a4e395bee8de5a9587dc7636eecf78c2988f477162a1f081c064","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-j22hl8\"\nUnexpected error:\n    \u003c*errors.joinError | 0x237cb4aa4138\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\": nodes \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x237cb4952040\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x237cb4952020\u003e{\n                    msg: \"failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x237cb4952000\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x237cb49b60a0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-j22hl8\"\nUnexpected error:\n    \u003c*errors.joinError | 0x237cb4aa4138\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\": nodes \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x237cb4952040\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x237cb4952020\u003e{\n                    msg: \"failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x237cb4952000\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x237cb49b60a0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\": nodes \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115090302321561600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"63ee057fa8457e9b8d70fd6b234bbed2bb83e872","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T14:36:39Z","started_at":"2026-10-27T14:36:39Z","completed_at":"2026-10-27T16:45:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T17:00:51.992252983Z","timing_metadata_checked_at":"2026-10-27T17:00:51.992252983Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T17:00:51.995503173Z","region_metadata_checked_at":"2026-10-27T17:00:51.995503173Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ac6bdc5408c9c528f8e884ed5e4cc3b4051d271ced98965392770951b21a7731","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115090302321561600","occurred_at":"2026-10-27T14:36:39Z","failed_at":"e2e","signature_id":"6ad8d70ef0c34180dc85202729c1b6c4224cb11128f69d82ce5269b20ff050bd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-94pk4p\"\nUnexpected error:\n    \u003c*errors.joinError | 0x380579046d8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\n    [pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-fj4vw/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x38056a72920\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                err: \u003c*fmt.wrapError | 0x38056fdc300\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x38056fb6b80\u003e{\n                        observation: \u003c*errors.errorString | 0x380577528d0\u003e{\n                            s: \"pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x38056fb6b20\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-fj4vw/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-94pk4p\"\nUnexpected error:\n    \u003c*errors.joinError | 0x380579046d8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\n    [pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-fj4vw/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x38056a72920\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                err: \u003c*fmt.wrapError | 0x38056fdc300\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x38056fb6b80\u003e{\n                        observation: \u003c*errors.errorString | 0x380577528d0\u003e{\n                            s: \"pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x38056fb6b20\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-fj4vw/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c8baf4b2e1a6c519c2e6a72beff1cb671e4fa39d2c9d8669c830590e4eb74e17","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node \u003cnode\u003e (reason=ProvisioningFailed, count=8, age=7m53s)","search_query":"[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c8baf4b2e1a6c519c2e6a72beff1cb671e4fa39d2c9d8669c830590e4eb74e17"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7774/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115090244503080960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7774,"pr_state":"closed","pr_sha":"b49980042db935724dae8d31dad5ecd613459d16","final_merged_sha":"b49980042db935724dae8d31dad5ecd613459d16","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T14:36:25Z","started_at":"2026-10-27T14:36:25Z","completed_at":"2026-10-27T16:38:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.861781634Z","timing_metadata_checked_at":"2026-10-27T16:50:27.861781634Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.864577625Z","region_metadata_checked_at":"2026-10-27T16:50:27.864577625Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115087800381476864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"f483b9230a17a45af1a3b7c16e817fe0969356bc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T14:26:42Z","started_at":"2026-10-27T14:26:42Z","completed_at":"2026-10-27T15:37:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:28:44.174535088Z","timing_metadata_checked_at":"2026-10-27T16:28:44.174535088Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:28:44.177982879Z","region_metadata_checked_at":"2026-10-27T16:28:44.177982879Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4db4cea81ad1c8d82f9e2a3d76d94024cc3a1a151aef7b2a407c7ec8da231fcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115087800381476864","occurred_at":"2026-10-27T14:26:42Z","failed_at":"alert","signature_id":"1dedc5282ccef457cd2bfffb95d080a0e8a4f79e607e2268fb8bfe663ae779f0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (12 unknown, 1 known)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:44:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75avtj8u4futjt2om8gigffgddu35-ingress-cert-le\", pod=\"openshift-apiserver-5fbf969589-rwzww\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75avtj8u4futjt2om8gigffgddu35-ingress-cert-le/openshift-apiserver-5fbf969589-rwzww has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:44:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn74o7qhu6go8i6sdevboooi4cvm5o8-o9s7l1c0u6a0c2b\", pod=\"openshift-apiserver-7c75f5b8c-869mc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn74o7qhu6go8i6sdevboooi4cvm5o8-o9s7l1c0u6a0c2b/openshift-apiserver-7c75f5b8c-869mc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75q9b8j8s3323r8hbovnl3fa3ah93-a8h6t7r0k6i7a1w\", pod=\"openshift-apiserver-6cf589c77c-9cn7v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75q9b8j8s3323r8hbovnl3fa3ah93-a8h6t7r0k6i7a1w/openshift-apiserver-6cf589c77c-9cn7v has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn74rifog60es9rvt2t2mfrpvs36h9e-t2y6g6x3g9p9s5t\", pod=\"openshift-apiserver-9c9cfdf89-62q4b\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn74rifog60es9rvt2t2mfrpvs36h9e-t2y6g6x3g9p9s5t/openshift-apiserver-9c9cfdf89-62q4b has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn757i0j3orr665p0dcqg487jcvp9a9-q1n4k4z8u2y9x7u\", pod=\"openshift-apiserver-7bbccfbcd8-nj4zh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn757i0j3orr665p0dcqg487jcvp9a9-q1n4k4z8u2y9x7u/openshift-apiserver-7bbccfbcd8-nj4zh has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75637s7ka00dgh3b0o5nl473kscrl-h2n8s4a0l1b6e9x\", pod=\"openshift-apiserver-7b79cf8d6d-klztg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75637s7ka00dgh3b0o5nl473kscrl-h2n8s4a0l1b6e9x/openshift-apiserver-7b79cf8d6d-klztg has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn7522f0rjmsl54mp9vum53bfeis22t-x1e5u3y9j2l5a2u\", pod=\"openshift-apiserver-7d75b894c6-n4cmx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn7522f0rjmsl54mp9vum53bfeis22t-x1e5u3y9j2l5a2u/openshift-apiserver-7d75b894c6-n4cmx has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75ndq1oh2entq0cv9ulrug5jog853-ingress-cert-le\", pod=\"openshift-apiserver-764b69ffcb-7ngbg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75ndq1oh2entq0cv9ulrug5jog853-ingress-cert-le/openshift-apiserver-764b69ffcb-7ngbg has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn7cebrepv6hitmognobm7c9cl2clnd-o5c0v9b6h0c5z0a\", pod=\"cluster-policy-controller-698bfd9c65-v6xdj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn7cebrepv6hitmognobm7c9cl2clnd-o5c0v9b6h0c5z0a/cluster-policy-controller-698bfd9c65-v6xdj has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75h85al80vo23t362mjtc3qgd04dp-c1b6v6c2x7c8p8f\", pod=\"openshift-apiserver-57bf4dc46f-h7mj5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75h85al80vo23t362mjtc3qgd04dp-c1b6v6c2x7c8p8f/openshift-apiserver-57bf4dc46f-h7mj5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn759i4ns4omt60u9uecngkrn5mj618-u8q4i0h7g0i1r6k\", pod=\"openshift-apiserver-59b86f6c67-76jwk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn759i4ns4omt60u9uecngkrn5mj618-u8q4i0h7g0i1r6k/openshift-apiserver-59b86f6c67-76jwk has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:47:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn7615u8n39pf897sksqb5f7beb9d2u-b0o3o9t7e5r0u9g\", pod=\"openshift-apiserver-77c77b89b8-kr9pl\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn7615u8n39pf897sksqb5f7beb9d2u-b0o3o9t7e5r0u9g/openshift-apiserver-77c77b89b8-kr9pl has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7776/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115086995314184192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7776,"pr_state":"closed","pr_sha":"b49ec264c195578785724fee9d83b480687a3925","final_merged_sha":"bb1de194f6d1f4d1c23b575387e90e2c886c823b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T14:23:30Z","started_at":"2026-10-27T14:23:30Z","completed_at":"2026-10-27T16:31:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.992764314Z","timing_metadata_checked_at":"2026-10-27T16:50:27.992764314Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.995757904Z","region_metadata_checked_at":"2026-10-27T16:50:27.995757904Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0bc595112701f5585e75e02aed9f95f9a53dfe8328530631f7e69a158a5fc8c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7776/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115086995314184192","occurred_at":"2026-10-27T14:23:30Z","failed_at":"alert","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T15:28:06Z\n  Ended:       2026-10-27T15:45:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4184192-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T15:28:06Z\n  Ended:       2026-10-27T15:45:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4184192-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085344536793088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7768,"pr_state":"closed","pr_sha":"169563406cb32ad3707f97b94c7a229632cbe344","final_merged_sha":"169563406cb32ad3707f97b94c7a229632cbe344","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T14:16:57Z","started_at":"2026-10-27T14:16:57Z","completed_at":"2026-10-27T16:37:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.481325055Z","timing_metadata_checked_at":"2026-10-27T16:50:27.481325055Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.484835744Z","region_metadata_checked_at":"2026-10-27T16:50:27.484835744Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085321392623616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7740,"pr_state":"closed","pr_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","final_merged_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T14:16:50Z","started_at":"2026-10-27T14:16:50Z","completed_at":"2026-10-27T16:39:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.329134344Z","timing_metadata_checked_at":"2026-10-27T16:50:27.329134344Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.332179334Z","region_metadata_checked_at":"2026-10-27T16:50:27.332179334Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085319714902016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7769,"pr_state":"closed","pr_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","final_merged_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T14:16:47Z","started_at":"2026-10-27T14:16:47Z","completed_at":"2026-10-27T16:34:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.606902352Z","timing_metadata_checked_at":"2026-10-27T16:50:27.606902352Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.609668443Z","region_metadata_checked_at":"2026-10-27T16:50:27.609668443Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d20af333c8424cd691265a5822df3f279017ff40c03b9a95aee64f864a9d7b2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085319714902016","occurred_at":"2026-10-27T14:16:47Z","failed_at":"e2e","signature_id":"f3dc95e2199f380975340c82cae7dc3176d54456533ece2dce8ed30b45782845","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfe103ee9c0\u003e: \n    failed to create NodePool np-m4xxs2: failed waiting for nodepool=\"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\" in resourcegroup=\"rg-candidate-4-22-9hlfn44n2sw7\" to finish creating: GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"name\": \"fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T15:26:37.894672503Z\",\n      \"endTime\": \"2026-10-27T15:45:41.452550561Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Compute/virtualMachines/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Network/networkInterfaces/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n            correlation ID: c84286ce-64a0-40f9-bed8-eadb6c3c019f\n            client request ID: 39555e0e-ee7f-4ff0-9056-9ab0bae5c62d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfe103ee9c0\u003e: \n    failed to create NodePool np-m4xxs2: failed waiting for nodepool=\"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\" in resourcegroup=\"rg-candidate-4-22-9hlfn44n2sw7\" to finish creating: GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"name\": \"fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T15:26:37.894672503Z\",\n      \"endTime\": \"2026-10-27T15:45:41.452550561Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Compute/virtualMachines/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Network/networkInterfaces/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n            correlation ID: c84286ce-64a0-40f9-bed8-eadb6c3c019f\n            client request ID: 39555e0e-ee7f-4ff0-9056-9ab0bae5c62d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6df737c7181b1e617209864731522070f8aa374ebcc148681c0698c5ad2ea926","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6df737c7181b1e617209864731522070f8aa374ebcc148681c0698c5ad2ea926"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7773/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085273577558016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7773,"pr_state":"closed","pr_sha":"3202579c21ae06689c691dcaa4cf116b00b2d5a1","final_merged_sha":"3202579c21ae06689c691dcaa4cf116b00b2d5a1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T14:16:38Z","started_at":"2026-10-27T14:16:38Z","completed_at":"2026-10-27T16:36:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:50:27.743094215Z","timing_metadata_checked_at":"2026-10-27T16:50:27.743094215Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:50:27.745933606Z","region_metadata_checked_at":"2026-10-27T16:50:27.745933606Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085218409877504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T14:16:27Z","started_at":"2026-10-27T14:16:27Z","completed_at":"2026-10-27T15:17:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T15:25:15.990343866Z","timing_metadata_checked_at":"2026-10-27T15:25:15.990343866Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T15:25:15.993764956Z","region_metadata_checked_at":"2026-10-27T15:25:15.993764956Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"48fa809ef6c7de1ddf9087bb6283f52da13155c4a1b50e34abbfc44280a28567","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085218409877504","occurred_at":"2026-10-27T14:16:27Z","failed_at":"alert","signature_id":"38d8ce206a673cd02519d29467557fc74a9f952f547b41255aa497ffa5a704f3","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-controller-859484fc49-qhntv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/ocm-controller-859484fc49-qhntv has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"cluster-manager-c5d6c587f-fd8rf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/cluster-manager-c5d6c587f-fd8rf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"multicluster-engine-operator-79c768d748-lg8vd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-lg8vd has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-applier\", pod=\"kube-applier-8475cd7c99-dfzv8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-applier/kube-applier-8475cd7c99-dfzv8 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-webhook-68676899fd-mklv5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/ocm-webhook-68676899fd-mklv5 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6dc8498b6f-ntd7x\", region=\"westus3\", severity=\"warning\"\n  Description: Pod mgmt-agent/mgmt-agent-6dc8498b6f-ntd7x has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"managedcluster-import-controller-v2-6cb46794f7-bzb25\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-bzb25 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-9hg5c\", region=\"westus3\", severity=\"warning\"\n  Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-9hg5c has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"5e3e7bb5166d13bdd575196924fea2923b9a46dd13757962f36b06cf79c60315","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085218409877504","occurred_at":"2026-10-27T14:16:27Z","failed_at":"provision","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T15:06:00.351Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-27T15:06:00.927Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:72abeb657cfbdd6a5891fc102a16429875d70c5932ad5994524d0246db914013 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:55a0ad895a8f10193bb070883194449f5252c057353887af1ba0767760a0d2f0 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-27T15:06:00.927Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-27T15:06:01.327Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-27T15:06:01.636Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-27T15:06:01.636Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-27T15:06:01.765Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-27T15:11:07.858Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-27T15:11:07.859Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115082291888787456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7770,"pr_state":"closed","pr_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","final_merged_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T14:04:49Z","started_at":"2026-10-27T14:04:49Z","completed_at":"2026-10-27T16:16:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T16:28:44.283691898Z","timing_metadata_checked_at":"2026-10-27T16:28:44.283691898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T16:28:44.28680759Z","region_metadata_checked_at":"2026-10-27T16:28:44.28680759Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cadb24487ed25d104465231daaa439634d01862e235259d3ca2d4f439360324c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115082291888787456","occurred_at":"2026-10-27T14:04:49Z","failed_at":"e2e","signature_id":"32ca328ce4f4d1f24d28c583c2c75a25c3a62e05b6f6a593c0daa6efba4fd145","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-sbw4s5\"\nUnexpected error:\n    \u003c*errors.joinError | 0xae6c2dd4150\u003e: \n    VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xae6c30c2320\u003e{\n                msg: \"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0xae6c3360200\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0xae6c335a000\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-sbw4s5\"\nUnexpected error:\n    \u003c*errors.joinError | 0xae6c2dd4150\u003e: \n    VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xae6c30c2320\u003e{\n                msg: \"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0xae6c3360200\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0xae6c335a000\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0c9a69ce334c27525179441c4e4ad72c2ac5b4078f88ec831ce8d15009143a74","failure_pattern":"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod \u003cnamespace\u003e/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pod \u003cpod\u003e)","search_query":"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0c9a69ce334c27525179441c4e4ad72c2ac5b4078f88ec831ce8d15009143a74"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115082045850914816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7775,"pr_state":"closed","pr_sha":"69e00ae59e8fd535ee9bdb8882209052cf6be2fd","final_merged_sha":"3b3384719c5a1520aba233f24370f02add587314","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T14:03:50Z","started_at":"2026-10-27T14:03:50Z","completed_at":"2026-10-27T14:20:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T14:33:28.695126435Z","timing_metadata_checked_at":"2026-10-27T14:33:28.695126435Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-27T14:33:28.69930012Z","region_metadata_checked_at":"2026-10-27T15:36:05.551929486Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d6f85fe20b39c7a2bf6593dd61770680b189dc8f29218aa2519a8ff7de2746ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115082045850914816","occurred_at":"2026-10-27T14:03:50Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069925570646016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T13:15:40Z","started_at":"2026-10-27T13:15:40Z","completed_at":"2026-10-27T15:33:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T15:47:01.972157137Z","timing_metadata_checked_at":"2026-10-27T15:47:01.972157137Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T15:47:01.975674725Z","region_metadata_checked_at":"2026-10-27T15:47:01.975674725Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069792967725056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6635,"pr_state":"closed","pr_sha":"4efb550d50b2877187fd7812a0fd49d4ba37d42a","final_merged_sha":"4efb550d50b2877187fd7812a0fd49d4ba37d42a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T13:15:09Z","started_at":"2026-10-27T13:15:09Z","completed_at":"2026-10-27T15:37:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T15:47:01.879767834Z","timing_metadata_checked_at":"2026-10-27T15:47:01.879767834Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T15:47:01.883530422Z","region_metadata_checked_at":"2026-10-27T15:47:01.883530422Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T13:15:01Z","started_at":"2026-10-27T13:15:01Z","completed_at":"2026-10-27T15:24:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T15:36:05.454736498Z","timing_metadata_checked_at":"2026-10-27T15:36:05.454736498Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T15:36:05.458659285Z","region_metadata_checked_at":"2026-10-27T15:36:05.458659285Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"5c982eca47b6a664590a7ec88a300f5682a999998dc20315b3ab668716be9a44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","occurred_at":"2026-10-27T13:15:01Z","failed_at":"alert","signature_id":"c57a7b8056a3891d85d45284445adb09d9edf7c2d1c098329ce32936845aefb2","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:32:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:33:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c9a58f81c420801807874bf7a156a6314bafec3e5c80821c2265483fd1da55ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","occurred_at":"2026-10-27T13:15:01Z","failed_at":"alert","signature_id":"d6ade9de695f38631520c2d467f5de31adc88dacbcbba03901a349e2aae5baa8","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:08:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-58f4f6fb5b-wt5ns\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-58f4f6fb5b-wt5ns has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"f910ad7264c6a4096f4e1559a8adae07440899a12eb10cdbec75f502fda30d37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","occurred_at":"2026-10-27T13:15:01Z","failed_at":"alert","signature_id":"7442bf20121518a5319998062acd103d25b97d6fedb3ac8a1e6323c1a65edc4c","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:42:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:43:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069745341403136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6954,"pr_state":"open","pr_sha":"74589019c042c52fc805b7e31036c33431fb627d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T13:14:57Z","started_at":"2026-10-27T13:14:57Z","completed_at":"2026-10-27T14:22:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T14:33:28.539070395Z","timing_metadata_checked_at":"2026-10-27T14:33:28.539070395Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T14:33:28.543077581Z","region_metadata_checked_at":"2026-10-27T14:33:28.543077581Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e4774f6542f4adf41a678929f57051826e63064315f95540129ce9207b618bd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069745341403136","occurred_at":"2026-10-27T13:14:57Z","failed_at":"alert","signature_id":"78df0de3512493987330c17664e0954e9157f8c822af2c10ada0cac833eb8371","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:13:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1403136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-58f4f6fb5b-kpvv8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-58f4f6fb5b-kpvv8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115063427364818944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T12:49:51Z","started_at":"2026-10-27T12:49:51Z","completed_at":"2026-10-27T13:49:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T14:02:21.323651402Z","timing_metadata_checked_at":"2026-10-27T14:02:21.323651402Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T14:02:21.326874691Z","region_metadata_checked_at":"2026-10-27T14:02:21.326874691Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"54b9ba1853c5931b398f8ae0520a1a914b18402579fda112be20c3095559a8d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115063427364818944","occurred_at":"2026-10-27T12:49:51Z","failed_at":"provision","signature_id":"6226765797caa992638fa67211f61137351872a74ebe70cc5200677de47ce171","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T13:38:02.530Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-10-27T13:38:03.129Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-10-27T13:38:03.129Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-27T13:38:03.342Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 namespace=multicluster-engine\ntime=2026-10-27T13:38:03.472Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 dryRun=false\ntime=2026-10-27T13:38:03.472Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-27T13:38:03.599Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-27T13:43:12.521Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-27T13:43:12.521Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c2e7122f012fcda98e11e9f11c6eb72558008d2e2cc0835ce25a3969a080ede6","failure_pattern":"failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded"}},{"row_id":"8406e7b2cd79883411cab7cc5ebf3a11a7254a9c329d6d6f8304ff83f7de39d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115063427364818944","occurred_at":"2026-10-27T12:49:51Z","failed_at":"alert","signature_id":"7ac72e7ab408d127706a4fc13bb3def720f1a28e42e40b0f980c057f98624718","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T13:44:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4818944-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"multicluster-engine-operator-79c768d748-6x98s\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-6x98s has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T13:44:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4818944-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-applier\", pod=\"kube-applier-5c96654689-vq66j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-applier/kube-applier-5c96654689-vq66j has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T13:44:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4818944-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-84bcf74974-q5fv9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod mgmt-agent/mgmt-agent-84bcf74974-q5fv9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","c2e7122f012fcda98e11e9f11c6eb72558008d2e2cc0835ce25a3969a080ede6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7755/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115061887891673088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7755,"pr_state":"closed","pr_sha":"a1e0f6f839aaa629e285b4f0a328e73b7e407bea","final_merged_sha":"a1e0f6f839aaa629e285b4f0a328e73b7e407bea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T12:43:44Z","started_at":"2026-10-27T12:43:44Z","completed_at":"2026-10-27T14:56:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T15:14:45.581027425Z","timing_metadata_checked_at":"2026-10-27T15:14:45.581027425Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T15:14:45.584985912Z","region_metadata_checked_at":"2026-10-27T15:14:45.584985912Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"78f803ab03399205f35b9d07cfc0fd49ffba77ed44e99b8308fde3a0b6d7b3a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7755/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115061887891673088","occurred_at":"2026-10-27T12:43:44Z","failed_at":"e2e","signature_id":"54d56dfc4a800362ca3c14d435d3f0db9206680219914967c341326c8f6b19e0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-g455dk\"\nUnexpected error:\n    \u003c*errors.joinError | 0x366719114510\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\n    [pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-7vqg4/e2e-csi-pvc\" (reason=Provisioning, count=10, age=44s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x366718da0b80\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                err: \u003c*fmt.wrapError | 0x366718677020\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x366718da09e0\u003e{\n                        observation: \u003c*errors.errorString | 0x366717e20d40\u003e{\n                            s: \"pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x366718da09c0\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-7vqg4/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-g455dk\"\nUnexpected error:\n    \u003c*errors.joinError | 0x366719114510\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\n    [pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-7vqg4/e2e-csi-pvc\" (reason=Provisioning, count=10, age=44s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x366718da0b80\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                err: \u003c*fmt.wrapError | 0x366718677020\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x366718da09e0\u003e{\n                        observation: \u003c*errors.errorString | 0x366717e20d40\u003e{\n                            s: \"pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x366718da09c0\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-7vqg4/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a8095a8b6eafff92d8870acf81e6750b45dca434bf91a8acb825aeaecf138193","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)","search_query":"[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a8095a8b6eafff92d8870acf81e6750b45dca434bf91a8acb825aeaecf138193"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115052847685963776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7373,"pr_state":"closed","pr_sha":"5051cc737d55c4b05cc92fa959d0f44fe99d3db6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T12:07:44Z","started_at":"2026-10-27T12:07:44Z","completed_at":"2026-10-27T14:26:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T14:43:28.788051788Z","timing_metadata_checked_at":"2026-10-27T14:43:28.788051788Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T14:43:28.791556176Z","region_metadata_checked_at":"2026-10-27T14:43:28.791556176Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"414c172c934c3f02c5aeff2707979c53c64e7314dce5e80773a5372082f6d6ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115052847685963776","occurred_at":"2026-10-27T12:07:44Z","failed_at":"e2e","signature_id":"709a8c51e125e721b929c34d4ce6d33c0724708a7660f16ba59386319e0d5760","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-9pvn6m\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6c70d0280d8\u003e: \n    VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) timed out after 10m0.004s: DNS A record \"*.e2e-dns-fs2vtm\" not yet present in zone rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-fs2vtm' does not exist in resource group 'rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n            correlation ID: 850b8d7d-f572-4c32-8b7c-4052d22cd119\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-9pvn6m\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6c70d0280d8\u003e: \n    VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) timed out after 10m0.004s: DNS A record \"*.e2e-dns-fs2vtm\" not yet present in zone rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-fs2vtm' does not exist in resource group 'rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n            correlation ID: 850b8d7d-f572-4c32-8b7c-4052d22cd119\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e2f2ef9a0323545de216b10f60b08cc70ff2c858dbbd51d763ae3430b634070","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-fs2vtm' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8e2f2ef9a0323545de216b10f60b08cc70ff2c858dbbd51d763ae3430b634070"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115052100529426432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"closed","pr_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","final_merged_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T12:04:51Z","started_at":"2026-10-27T12:04:51Z","completed_at":"2026-10-27T13:57:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T14:13:01.698952721Z","timing_metadata_checked_at":"2026-10-27T14:13:01.698952721Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T14:13:01.702631909Z","region_metadata_checked_at":"2026-10-27T14:13:01.702631909Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115043139386347520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6954,"pr_state":"open","pr_sha":"69ab8aa9b4190c2ead4251af762ef5bfdfc1a437","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T11:29:14Z","started_at":"2026-10-27T11:29:14Z","completed_at":"2026-10-27T12:53:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:10:25.361900297Z","timing_metadata_checked_at":"2026-10-27T13:10:25.361900297Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:10:25.365555485Z","region_metadata_checked_at":"2026-10-27T13:10:25.365555485Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b115375c07628c9d8348c63227efbe6847dfeb25d59345a15b4d43602d4a854d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115043139386347520","occurred_at":"2026-10-27T11:29:14Z","failed_at":"e2e","signature_id":"d7f51ef3f79440dd76467c8a42d472c3bd9858d482536084ca3eb59fd60d6127","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:117]: Insights Operator was not deployed or did not reach Disabled/NoToken state\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa7745440900\u003e: \n    VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()\n    {\n        msg: \"VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n        err: \u003c*verifiers.pollObservationError | 0xa77454408e0\u003e{\n            observation: \u003c*errors.errorString | 0xa77456c4580\u003e{\n                s: \"ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xa77454408c0\u003e{\n                msg: \"failed to get ClusterOperator \\\"insights\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:117]: Insights Operator was not deployed or did not reach Disabled/NoToken state\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa7745440900\u003e: \n    VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()\n    {\n        msg: \"VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n        err: \u003c*verifiers.pollObservationError | 0xa77454408e0\u003e{\n            observation: \u003c*errors.errorString | 0xa77456c4580\u003e{\n                s: \"ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xa77454408c0\u003e{\n                msg: \"failed to get ClusterOperator \\\"insights\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df10c7ad9b6f58b98b74568dc817b34eae1f531ef761f38ba689e30339ed72f4","failure_pattern":"VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df10c7ad9b6f58b98b74568dc817b34eae1f531ef761f38ba689e30339ed72f4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040287725195264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7769,"pr_state":"closed","pr_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","final_merged_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T11:17:54Z","started_at":"2026-10-27T11:17:54Z","completed_at":"2026-10-27T13:42:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:52:00.882609004Z","timing_metadata_checked_at":"2026-10-27T13:52:00.882609004Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:52:00.885794893Z","region_metadata_checked_at":"2026-10-27T13:52:00.885794893Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"5eac7102213294d6ca6b1801d0e30ecd28ac1a3c9656d2cdf0933016cbb66b12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040287725195264","occurred_at":"2026-10-27T11:17:54Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T12:39:32Z\n  Ended:       2026-10-27T12:54:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5195264-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7770,"pr_state":"closed","pr_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","final_merged_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T11:17:21Z","started_at":"2026-10-27T11:17:21Z","completed_at":"2026-10-27T13:35:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:52:01.267410951Z","timing_metadata_checked_at":"2026-10-27T13:52:01.267410951Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:52:01.270697641Z","region_metadata_checked_at":"2026-10-27T13:52:01.270697641Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"864921c8fae871c96e848f5c004b6eaace0ec9acfe98e8dc94662017ac1671a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","occurred_at":"2026-10-27T11:17:21Z","failed_at":"e2e","signature_id":"526f904cbbc8ad7301bdb1bf3d4e1debe9adef61ef836cb69e29184539b3a4f6","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-lh6qk9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fdd98ed4800\u003e: \n    DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3fdd9882d200\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-lh6qk9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fdd98ed4800\u003e: \n    DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3fdd9882d200\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"92bc9a0ad9087924a191e4a5c284d84d336991bb05746bd22d381ac64b3463f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","occurred_at":"2026-10-27T11:17:21Z","failed_at":"e2e","signature_id":"0b71c966ba6609560492817e57b68323576e947b7f4d9a9c9eeb5fc7aa08c840","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-hd6rvlcsz446/cluster-candidate-4-23-z5pr5h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd0b2342e360\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-z5pr5h: failed waiting for cluster=\"cluster-candidate-4-23-z5pr5h\" in resourcegroup=\"rg-candidate-4-23-hd6rvlcsz446\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"name\": \"7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:16:28.588568581Z\",\n      \"endTime\": \"2026-10-27T12:27:01.432725234Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n            correlation ID: 3d3fc710-9f38-4fa0-b097-667c1de10d30\n            client request ID: 7b1eba2b-65de-404e-b3ec-50e9f1177df4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-hd6rvlcsz446/cluster-candidate-4-23-z5pr5h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd0b2342e360\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-z5pr5h: failed waiting for cluster=\"cluster-candidate-4-23-z5pr5h\" in resourcegroup=\"rg-candidate-4-23-hd6rvlcsz446\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"name\": \"7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:16:28.588568581Z\",\n      \"endTime\": \"2026-10-27T12:27:01.432725234Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n            correlation ID: 3d3fc710-9f38-4fa0-b097-667c1de10d30\n            client request ID: 7b1eba2b-65de-404e-b3ec-50e9f1177df4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e46fad7d5fafea60a6252610a97029d6a9421cce4f39dc628483a7323d419f95","failure_pattern":"ERROR CODE: InvalidResource; detail message DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"fcdec00e2746b4566a3e73d29cd1321ab1b53c1b48466ea38989c8b04a8b8e1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","occurred_at":"2026-10-27T11:17:21Z","failed_at":"e2e","signature_id":"b76a2eb0d9fc40e5751f44dcc0f4dac27426fa6827332cd4c74ac4ab6381cab3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-22-gcsfgvdh5qhb/cluster-candidate-4-22-2w4gz9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30d8f6930800\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-2w4gz9: failed waiting for cluster=\"cluster-candidate-4-22-2w4gz9\" in resourcegroup=\"rg-candidate-4-22-gcsfgvdh5qhb\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"name\": \"74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:17:08.196779757Z\",\n      \"endTime\": \"2026-10-27T12:28:21.232176744Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n            correlation ID: d480a26c-43a2-49b4-8de4-02fc3c46d885\n            client request ID: 8e7d5833-9216-45e5-8c64-c9b98a48c0a2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-22-gcsfgvdh5qhb/cluster-candidate-4-22-2w4gz9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30d8f6930800\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-2w4gz9: failed waiting for cluster=\"cluster-candidate-4-22-2w4gz9\" in resourcegroup=\"rg-candidate-4-22-gcsfgvdh5qhb\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"name\": \"74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:17:08.196779757Z\",\n      \"endTime\": \"2026-10-27T12:28:21.232176744Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n            correlation ID: d480a26c-43a2-49b4-8de4-02fc3c46d885\n            client request ID: 8e7d5833-9216-45e5-8c64-c9b98a48c0a2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e46fad7d5fafea60a6252610a97029d6a9421cce4f39dc628483a7323d419f95","failure_pattern":"ERROR CODE: InvalidResource; detail message DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","e46fad7d5fafea60a6252610a97029d6a9421cce4f39dc628483a7323d419f95"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7740,"pr_state":"closed","pr_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","final_merged_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T11:16:57Z","started_at":"2026-10-27T11:16:57Z","completed_at":"2026-10-27T13:38:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:51:59.727074265Z","timing_metadata_checked_at":"2026-10-27T13:51:59.727074265Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:51:59.730157555Z","region_metadata_checked_at":"2026-10-27T13:51:59.730157555Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1eb4f83fb58966734f4f270982dc6fbdd9397c85f4da16b9cc632ded2d0dc71c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","occurred_at":"2026-10-27T11:16:57Z","failed_at":"e2e","signature_id":"66506202a3c311949180fc813c9fe22cf549fe6653665a5b8f7d13e545e85d6a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-jhb5b2\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32758ffae460\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-jhb5b2: failed waiting for cluster=\"cluster-zstream-4-22-jhb5b2\" in resourcegroup=\"rg-zstream-upgrade-4-22-lhz7fdj9k86z\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"name\": \"fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:17.939641835Z\",\n      \"endTime\": \"2026-10-27T12:30:15.55604778Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n            correlation ID: 046c9d63-6366-4a78-bd4e-177dad3c7de6\n            client request ID: 40011f58-fbc8-44b5-a868-12a5ed482aad\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-jhb5b2\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32758ffae460\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-jhb5b2: failed waiting for cluster=\"cluster-zstream-4-22-jhb5b2\" in resourcegroup=\"rg-zstream-upgrade-4-22-lhz7fdj9k86z\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"name\": \"fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:17.939641835Z\",\n      \"endTime\": \"2026-10-27T12:30:15.55604778Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n            correlation ID: 046c9d63-6366-4a78-bd4e-177dad3c7de6\n            client request ID: 40011f58-fbc8-44b5-a868-12a5ed482aad\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e46fad7d5fafea60a6252610a97029d6a9421cce4f39dc628483a7323d419f95","failure_pattern":"ERROR CODE: InvalidResource; detail message DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"b7bc1fabc7177724c95fb03eb15591de000154ad5820f3dd906ca6c211dc3b82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","occurred_at":"2026-10-27T11:16:57Z","failed_at":"e2e","signature_id":"82b8f0fbf27dd3db5294e829df65daa69cb7944c0cc4bdd9e9e49cf2f699cb72","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x45c8638e340\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5vfww4h6t295\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"name\": \"1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:13.19675558Z\",\n      \"endTime\": \"2026-10-27T12:30:26.100595452Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n            correlation ID: 724394e1-dd61-4ca3-aed0-37913a4e01f3\n            client request ID: f5bcc160-8ac9-4bb8-b98f-d16472eb1f7c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x45c8638e340\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5vfww4h6t295\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"name\": \"1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:13.19675558Z\",\n      \"endTime\": \"2026-10-27T12:30:26.100595452Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n            correlation ID: 724394e1-dd61-4ca3-aed0-37913a4e01f3\n            client request ID: f5bcc160-8ac9-4bb8-b98f-d16472eb1f7c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a7b7935e0fbe062a83ac94a424d97876ea2ab08c4e53e37b439fcd600a57bd3","failure_pattern":"ERROR CODE: InvalidResource; detail message ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"ffe1cdb891ddf8cfd230e64f626f048eb837e377b69b02165bee67031b05a62c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","occurred_at":"2026-10-27T11:16:57Z","failed_at":"e2e","signature_id":"55903a7e057966beaff233554a1a3ab22e6bb5909a2a13f86577c189bd43494f","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1523283f0200\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-dzpqr982szjp\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"name\": \"5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:12.67761363Z\",\n      \"endTime\": \"2026-10-27T12:30:55.93851152Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n            correlation ID: d35a35bb-1a40-4d4f-9155-942bc6b994d5\n            client request ID: 483339d8-63cb-4cfb-a4bc-1a9aa9a36174\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1523283f0200\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-dzpqr982szjp\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"name\": \"5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:12.67761363Z\",\n      \"endTime\": \"2026-10-27T12:30:55.93851152Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n            correlation ID: d35a35bb-1a40-4d4f-9155-942bc6b994d5\n            client request ID: 483339d8-63cb-4cfb-a4bc-1a9aa9a36174\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a7b7935e0fbe062a83ac94a424d97876ea2ab08c4e53e37b439fcd600a57bd3","failure_pattern":"ERROR CODE: InvalidResource; detail message ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1a7b7935e0fbe062a83ac94a424d97876ea2ab08c4e53e37b439fcd600a57bd3","e46fad7d5fafea60a6252610a97029d6a9421cce4f39dc628483a7323d419f95"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115039807775182848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7768,"pr_state":"closed","pr_sha":"169563406cb32ad3707f97b94c7a229632cbe344","final_merged_sha":"169563406cb32ad3707f97b94c7a229632cbe344","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T11:16:00Z","started_at":"2026-10-27T11:16:00Z","completed_at":"2026-10-27T13:38:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:52:00.556926664Z","timing_metadata_checked_at":"2026-10-27T13:52:00.556926664Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:52:00.559931354Z","region_metadata_checked_at":"2026-10-27T13:52:00.559931354Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"97213af270f9df3c0bcdf850576974f7e0c5d8aa54ea88dde573c842f7c0619f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115039807775182848","occurred_at":"2026-10-27T11:16:00Z","failed_at":"e2e","signature_id":"09bf758da913ed0a3cc99ca9f1bd9cf87fbe7bfffcfae83211bf1e062a809761","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:108]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-23-4q7sfm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1abe3e2ce320\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2\" in resourcegroup=\"rg-zstream-upgrade-4-23-r8wjcwsdhsgm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n            correlation ID: adb21106-8945-4c8c-8718-f7a2e847f630\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:108]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-23-4q7sfm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1abe3e2ce320\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2\" in resourcegroup=\"rg-zstream-upgrade-4-23-r8wjcwsdhsgm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n            correlation ID: adb21106-8945-4c8c-8718-f7a2e847f630\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115038760361005056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7757,"pr_state":"open","pr_sha":"890e4b7ad12612a754ecb680230c60623eddeac7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T11:11:50Z","started_at":"2026-10-27T11:11:50Z","completed_at":"2026-10-27T13:41:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:52:00.110999915Z","timing_metadata_checked_at":"2026-10-27T13:52:00.110999915Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:52:00.114037505Z","region_metadata_checked_at":"2026-10-27T13:52:00.114037505Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115036855358459904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7771,"pr_state":"open","pr_sha":"484af29e198fbca448bbd75e5a5bbb829d350783","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T11:04:06Z","started_at":"2026-10-27T11:04:06Z","completed_at":"2026-10-27T13:37:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:52:01.689250978Z","timing_metadata_checked_at":"2026-10-27T13:52:01.689250978Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:52:01.69189937Z","region_metadata_checked_at":"2026-10-27T13:52:01.69189937Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8153100da5fb70874c24bf194c13d05de472489edbd60b2d419d22238b672d04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115036855358459904","occurred_at":"2026-10-27T11:04:06Z","failed_at":"e2e","signature_id":"55f92aabdfd5dc2b9a9243a121a5e6b1cc2cbbedea71dec47917cd4058b78750","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-mf9p48\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3fde6df0348\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\": nodes \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3fde61ae7e0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\": nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x3fde61ae7c0\u003e{\n                    msg: \"failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\": nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x3fde61ae7a0\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\": nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x3fde6bae3c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-mf9p48\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3fde6df0348\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\": nodes \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3fde61ae7e0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\": nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x3fde61ae7c0\u003e{\n                    msg: \"failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\": nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x3fde61ae7a0\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\": nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x3fde6bae3c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\": nodes \"n5m6j4y9e5r3n1x-np-mf9p48-5rm8z-btdr5\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7773/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115035820850155520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7773,"pr_state":"closed","pr_sha":"3202579c21ae06689c691dcaa4cf116b00b2d5a1","final_merged_sha":"3202579c21ae06689c691dcaa4cf116b00b2d5a1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T11:00:09Z","started_at":"2026-10-27T11:00:09Z","completed_at":"2026-10-27T13:19:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:31:33.343320979Z","timing_metadata_checked_at":"2026-10-27T13:31:33.343320979Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:31:33.346909267Z","region_metadata_checked_at":"2026-10-27T13:31:33.346909267Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"df7a9182fe3a48d1ad559fc6cba63df42e3065a17413f6a332223c7e6ed8efb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7773/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115035820850155520","occurred_at":"2026-10-27T11:00:09Z","failed_at":"e2e","signature_id":"16ae8d5d9b50963b314bb9a1ebf572dcd3f3137a44de5de8da13239140ef0aec","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17b1db069120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*errors.errorString | 0x17b1da3a61e0\u003e{\n            s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17b1db069120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*errors.errorString | 0x17b1da3a61e0\u003e{\n            s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6249e15d0a6c7cf0142092c6b9eca0eb8f84826824bb608613ac26e30d3b14b0","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6249e15d0a6c7cf0142092c6b9eca0eb8f84826824bb608613ac26e30d3b14b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115034279682510848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"closed","pr_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","final_merged_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T10:54:02Z","started_at":"2026-10-27T10:54:02Z","completed_at":"2026-10-27T11:32:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T11:46:29.659643023Z","timing_metadata_checked_at":"2026-10-27T11:46:29.659643023Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T11:46:29.663000812Z","region_metadata_checked_at":"2026-10-27T11:46:29.663000812Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b13c253a24d018eeaab937dde624bff7fc74373e0f4bd0ce8443a56a6412ca0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115034279682510848","occurred_at":"2026-10-27T10:54:02Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T11:21:11.139Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T11:21:12.548Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T11:21:13.677Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=7510a52b8b6967f369df752e1ecd7511738ad8092c8cb66a264114e05ac9f88f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2510848-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7510a52b8b6967f369df752e1ecd7511738ad8092c8cb66a264114e05ac9f88f\ntime=2026-10-27T11:23:14.094Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2510848-svc/providers/Microsoft.Resources/deployments/7510a52b8b6967f369df752e1ecd7511738ad8092c8cb66a264114e05ac9f88f/operationStatuses/08584085068123757360\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2510848-svc/providers/Microsoft.Resources/deployments/cluster-7vqst4vw3vptw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2510848-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2510848-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j2510848-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-27T11:25:14.100Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T11:25:14.853Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T11:25:16.187Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=d377865c9ece3412c41698f7290eb35643fec6eb6c74141ac09360de5e692c4b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2510848-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd377865c9ece3412c41698f7290eb35643fec6eb6c74141ac09360de5e692c4b\ntime=2026-10-27T11:26:46.433Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2510848-svc/providers/Microsoft.Resources/deployments/d377865c9ece3412c41698f7290eb35643fec6eb6c74141ac09360de5e692c4b/operationStatuses/08584085065700090638\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115031737661657088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"closed","pr_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","final_merged_sha":"809f0eceaea2f54166c93673b237f8553f4d1070","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T10:43:56Z","started_at":"2026-10-27T10:43:56Z","completed_at":"2026-10-27T12:58:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:10:25.383585124Z","timing_metadata_checked_at":"2026-10-27T13:10:25.383585124Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:10:25.392190595Z","region_metadata_checked_at":"2026-10-27T13:10:25.392190595Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e5793d0518cf351e40bb7504edc968929d0e70f91d795d427a83ec74a151e268","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115031737661657088","occurred_at":"2026-10-27T10:43:56Z","failed_at":"alert","signature_id":"232640154453a0f874248e0023fa85692463590a9f33b666fe19b05a6e973c88","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T12:07:21Z\n  Ended:       2026-10-27T12:13:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1657088-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn3rdsscm4njq5vp9j636vrehjrq243-d8n2y7y8n3i9x3v\", pod=\"cluster-policy-controller-fd98b88bf-g2g4g\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn3rdsscm4njq5vp9j636vrehjrq243-d8n2y7y8n3i9x3v/cluster-policy-controller-fd98b88bf-g2g4g has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115029873931063296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7739,"pr_state":"closed","pr_sha":"80c56ebc5ecebae5a5e813348043a2e9b349b351","final_merged_sha":"85f478eff30207b95d439e1d26791f75c7ff20f7","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T10:36:31Z","started_at":"2026-10-27T10:36:31Z","completed_at":"2026-10-27T12:43:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T13:00:01.494874315Z","timing_metadata_checked_at":"2026-10-27T13:00:01.494874315Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T13:00:01.498029504Z","region_metadata_checked_at":"2026-10-27T13:00:01.498029504Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115028748561551360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7658,"pr_state":"open","pr_sha":"79aa421cf5aace64032c519279c3c0d4c1870440","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-27T10:32:03Z","started_at":"2026-10-27T10:32:03Z","completed_at":"2026-10-27T12:29:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:49:28.17104175Z","timing_metadata_checked_at":"2026-10-27T12:49:28.17104175Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:49:28.174046341Z","region_metadata_checked_at":"2026-10-27T12:49:28.174046341Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115028667607289856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"59237734f16effd45120ec9d489f6cdac1d229a9","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T10:31:44Z","started_at":"2026-10-27T10:31:44Z","completed_at":"2026-10-27T11:36:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:17:29.066539873Z","timing_metadata_checked_at":"2026-10-27T12:17:29.066539873Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:17:29.069938861Z","region_metadata_checked_at":"2026-10-27T12:17:29.069938861Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4cff3d86e96693202c1e4cb204c3ce3998301cb387bb10edb628e0ab42097163","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115028667607289856","occurred_at":"2026-10-27T10:31:44Z","failed_at":"alert","signature_id":"836edef393ea769d8c1dbbd5c1835e62c2571df370eab8875c778843c628400d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T11:44:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7289856-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn3odumfrf0gec5eoopkp95nbd50npg-g7y4o6a1c7o1n4l\", pod=\"openshift-apiserver-fcbc8b75b-7bkvq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn3odumfrf0gec5eoopkp95nbd50npg-g7y4o6a1c7o1n4l/openshift-apiserver-fcbc8b75b-7bkvq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115026232805429248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"a630f5d29ba754ca53a2c8994a802571baafa9d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T10:22:03Z","started_at":"2026-10-27T10:22:03Z","completed_at":"2026-10-27T10:56:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T11:04:35.629499164Z","timing_metadata_checked_at":"2026-10-27T11:04:35.629499164Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T11:04:35.632692653Z","region_metadata_checked_at":"2026-10-27T11:04:35.632692653Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2e157a93eab00f9034b40035415ead5a0237ccb695e674ba8cbc50814a53cc36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115026232805429248","occurred_at":"2026-10-27T10:22:03Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T10:44:53.940Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T10:44:55.865Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T10:44:57.046Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=3a4e9448a9762a985cb316b4d8489410c3bbe88c9544a1ea43a43108af2bed28 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5429248-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3a4e9448a9762a985cb316b4d8489410c3bbe88c9544a1ea43a43108af2bed28\ntime=2026-10-27T10:47:27.463Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5429248-svc/providers/Microsoft.Resources/deployments/3a4e9448a9762a985cb316b4d8489410c3bbe88c9544a1ea43a43108af2bed28/operationStatuses/08584085089891732041\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5429248-svc/providers/Microsoft.Resources/deployments/cluster-xgba2pasuetkq\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5429248-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5429248-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j5429248-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-27T10:49:27.464Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T10:49:28.594Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T10:49:29.555Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=f09e3613965c5585c0ab1826e3f5464b156c22da3c4be22e911d26ff9225b208 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5429248-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff09e3613965c5585c0ab1826e3f5464b156c22da3c4be22e911d26ff9225b208\ntime=2026-10-27T10:50:59.825Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5429248-svc/providers/Microsoft.Resources/deployments/f09e3613965c5585c0ab1826e3f5464b156c22da3c4be22e911d26ff9225b208/operationStatuses/08584085087164463793\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025512358219776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T10:19:11Z","started_at":"2026-10-27T10:19:11Z","completed_at":"2026-10-27T12:25:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:38:30.747629931Z","timing_metadata_checked_at":"2026-10-27T12:38:30.747629931Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:38:30.752395015Z","region_metadata_checked_at":"2026-10-27T12:38:30.752395015Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"28b3c0560afe210aaf520a474e6dbf89dba8d8309216014e1c23094ae5372f99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025512358219776","occurred_at":"2026-10-27T10:19:11Z","failed_at":"e2e","signature_id":"2787f66c9d2a7a1f7a92b3a296589aebcd6aaf28beaeae1caae03bb3fb461214","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-wcdhz7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x13c32a30a4f8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-2drrj/e2e-csi-pvc\" (reason=Provisioning, count=10, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13c329ac69a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                err: \u003c*fmt.wrapError | 0x13c329f00fa0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x13c329f00aa0\u003e{\n                        observation: \u003c*errors.errorString | 0x13c329cc21b0\u003e{\n                            s: \"PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x13c329f00a60\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-2drrj/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-wcdhz7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x13c32a30a4f8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-2drrj/e2e-csi-pvc\" (reason=Provisioning, count=10, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13c329ac69a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                err: \u003c*fmt.wrapError | 0x13c329f00fa0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x13c329f00aa0\u003e{\n                        observation: \u003c*errors.errorString | 0x13c329cc21b0\u003e{\n                            s: \"PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x13c329f00a60\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-2drrj/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b616f5a6866cb92586bb6c324d97afd731ae9a87974504a140606af8f1c6d8c7","failure_pattern":"error code: CANCEL","search_query":"error code: CANCEL"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b616f5a6866cb92586bb6c324d97afd731ae9a87974504a140606af8f1c6d8c7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"5bbe82813cee485ba01f3d4a450f0a6b53de7593","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T10:19:07Z","started_at":"2026-10-27T10:19:07Z","completed_at":"2026-10-27T12:23:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:38:30.881008205Z","timing_metadata_checked_at":"2026-10-27T12:38:30.881008205Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:38:30.883981296Z","region_metadata_checked_at":"2026-10-27T12:38:30.883981296Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"669316fa9a83d335181d37ef5dbfae51bab3863261500f5ef7500dcf5b71a78d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","occurred_at":"2026-10-27T10:19:07Z","failed_at":"e2e","signature_id":"ace47547bb37bbb48ad9829ba0c4939a4085b942cd0b112d65b243d63eb8c4f2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.696s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x283ddd596200\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.696s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x283ddd596200\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87b6fa8784f23f8d704a5e8b3db46de48e9b95d9d70ba77988130bb26adb90a4","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0"}},{"row_id":"c5f7751fdf6153c9f49a1d9e3a00bf0b98048cfdcb1617f8dea55dcd7dad996d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","occurred_at":"2026-10-27T10:19:07Z","failed_at":"alert","signature_id":"4c14651213a33fb215c6ba3326174190c0648e3ebef08c277144d262c0761f29","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T11:22:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4445952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4445952/providers/microsoft.monitor/accounts/services-j4445952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-qj8wt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4196d6e7-f3b2-4df6-a959-ed3e1297c6ea\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-qj8wt (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T11:24:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4445952-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.27:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4445952/providers/microsoft.monitor/accounts/services-j4445952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-vpw6d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e6f828e4-a265-4c17-9265-1793badfcdf7\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-vpw6d (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T11:24:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4445952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4445952/providers/microsoft.monitor/accounts/services-j4445952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-qj8wt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4196d6e7-f3b2-4df6-a959-ed3e1297c6ea\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-qj8wt (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T11:24:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4445952-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.27:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4445952/providers/microsoft.monitor/accounts/services-j4445952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-vpw6d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e6f828e4-a265-4c17-9265-1793badfcdf7\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-vpw6d (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"d2e9ac80653c7a2c6a20583da400bd83c7a1fc38e8590d045f43af6f1f5513e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","occurred_at":"2026-10-27T10:19:07Z","failed_at":"e2e","signature_id":"d36af26ba43900f37b1a8934a7c91a186aab3388d1da85c98b9b4b581315a6c9","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10bb5e13c7a0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6gzvpqk2566b\" to finish creating: GET https://rp.j4445952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f17d3b43-5248-42e8-9f44-dbdc6056affd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f17d3b43-5248-42e8-9f44-dbdc6056affd\",\n      \"name\": \"f17d3b43-5248-42e8-9f44-dbdc6056affd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T11:25:28.537630373Z\",\n      \"endTime\": \"2026-10-27T11:54:32.228087726Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4445952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f17d3b43-5248-42e8-9f44-dbdc6056affd\n            correlation ID: bb86b9bc-b93e-4515-9de7-3f81a42f7835\n            client request ID: b2ae866e-4769-4b2c-931c-8d15a52ca9ea\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10bb5e13c7a0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6gzvpqk2566b\" to finish creating: GET https://rp.j4445952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f17d3b43-5248-42e8-9f44-dbdc6056affd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f17d3b43-5248-42e8-9f44-dbdc6056affd\",\n      \"name\": \"f17d3b43-5248-42e8-9f44-dbdc6056affd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T11:25:28.537630373Z\",\n      \"endTime\": \"2026-10-27T11:54:32.228087726Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4445952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f17d3b43-5248-42e8-9f44-dbdc6056affd\n            correlation ID: bb86b9bc-b93e-4515-9de7-3f81a42f7835\n            client request ID: b2ae866e-4769-4b2c-931c-8d15a52ca9ea\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","87b6fa8784f23f8d704a5e8b3db46de48e9b95d9d70ba77988130bb26adb90a4","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7761/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115023092878151680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7761,"pr_state":"closed","pr_sha":"55c5667b19edf70588255af6ff519fedfcc8ed78","final_merged_sha":"55c5667b19edf70588255af6ff519fedfcc8ed78","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T10:09:35Z","started_at":"2026-10-27T10:09:35Z","completed_at":"2026-10-27T12:15:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:28:13.598476669Z","timing_metadata_checked_at":"2026-10-27T12:28:13.598476669Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:28:13.601720459Z","region_metadata_checked_at":"2026-10-27T12:28:13.601720459Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019604114804736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6635,"pr_state":"closed","pr_sha":"4efb550d50b2877187fd7812a0fd49d4ba37d42a","final_merged_sha":"4efb550d50b2877187fd7812a0fd49d4ba37d42a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T09:55:43Z","started_at":"2026-10-27T09:55:43Z","completed_at":"2026-10-27T12:09:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:28:13.134869099Z","timing_metadata_checked_at":"2026-10-27T12:28:13.134869099Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:28:13.138771587Z","region_metadata_checked_at":"2026-10-27T12:28:13.138771587Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d3de06124f3ff46871191931a87d1e52bccf598e22e17ecd02071275f756a84c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019604114804736","occurred_at":"2026-10-27T09:55:43Z","failed_at":"e2e","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019570476486656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T09:55:35Z","started_at":"2026-10-27T09:55:35Z","completed_at":"2026-10-27T12:09:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:28:13.262391206Z","timing_metadata_checked_at":"2026-10-27T12:28:13.262391206Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:28:13.265851995Z","region_metadata_checked_at":"2026-10-27T12:28:13.265851995Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a482388282fb99597351bf33c05eec53eba0e4e442e117553aa54f0305f505c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019570476486656","occurred_at":"2026-10-27T09:55:35Z","failed_at":"e2e","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019495591383040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"16b8847bfaee5fa81fecaeaf4e97cfb93891dd36","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T09:55:17Z","started_at":"2026-10-27T09:55:17Z","completed_at":"2026-10-27T12:16:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T12:28:13.406062363Z","timing_metadata_checked_at":"2026-10-27T12:28:13.406062363Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T12:28:13.409398653Z","region_metadata_checked_at":"2026-10-27T12:28:13.409398653Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1e4cfde4aee7a5b6a5fe46857886adee91d8a5ddfab231154f92e3e0d859b8c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019495591383040","occurred_at":"2026-10-27T09:55:17Z","failed_at":"alert","signature_id":"e051fbf77b070edcec4d4461b179266cc25bf3709a7c61aa952c8a230f477902","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T11:32:01Z\n  Ended:       2026-10-27T11:38:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1383040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn3c1q684s9gqm9rpn1jj8tdrij98ii-cilium-cluster\", pod=\"kube-scheduler-cb7c4654b-xhqnb\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn3c1q684s9gqm9rpn1jj8tdrij98ii-cilium-cluster/kube-scheduler-cb7c4654b-xhqnb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008102154637312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7514,"pr_state":"open","pr_sha":"eb3b15b6eba57ec61003502a12cd451237522079","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T09:10:01Z","started_at":"2026-10-27T09:10:01Z","completed_at":"2026-10-27T11:14:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T11:26:01.707218273Z","timing_metadata_checked_at":"2026-10-27T11:26:01.707218273Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T11:26:01.710617062Z","region_metadata_checked_at":"2026-10-27T11:26:01.710617062Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"98851ad3ba4d9e317d2043f3aa68b59ebd81c88b8cb650a0cd42cb41cad4c975","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008102154637312","occurred_at":"2026-10-27T09:10:01Z","failed_at":"e2e","signature_id":"00fd1cc38488aefa9ab60f3c0d51de5fccf6401e991fc65c7e07a02bea28a323","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2874138e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*errors.errorString | 0x2874134f7c90\u003e{\n            s: \"dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2874138e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*errors.errorString | 0x2874134f7c90\u003e{\n            s: \"dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a9a6f789dd8ac4b0a8bb332fa94ef06d366fbcb926203ea587fdddca5af97929","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}},{"row_id":"c1208ad410040193753ecfbe2ec642546c0608a785ce18802450b17f8ce2dad8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008102154637312","occurred_at":"2026-10-27T09:10:01Z","failed_at":"e2e","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","a9a6f789dd8ac4b0a8bb332fa94ef06d366fbcb926203ea587fdddca5af97929"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008051160289280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7770,"pr_state":"closed","pr_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","final_merged_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T09:09:48Z","started_at":"2026-10-27T09:09:48Z","completed_at":"2026-10-27T11:08:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T11:15:17.126856837Z","timing_metadata_checked_at":"2026-10-27T11:15:17.126856837Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T11:15:17.129704428Z","region_metadata_checked_at":"2026-10-27T11:15:17.129704428Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4360c4be1a886c9dde53f8ee305f945fe6015399d0361ff4b849355d448a1e0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008051160289280","occurred_at":"2026-10-27T09:09:48Z","failed_at":"e2e","signature_id":"dce5e5ce5976358e3c7a925a1fbd158f7e68fc614b9e70512494514a648c8b2a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114994824758169600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7740,"pr_state":"closed","pr_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","final_merged_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T08:17:15Z","started_at":"2026-10-27T08:17:15Z","completed_at":"2026-10-27T10:16:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T10:22:34.259483926Z","timing_metadata_checked_at":"2026-10-27T10:22:34.259483926Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T10:22:34.265383407Z","region_metadata_checked_at":"2026-10-27T10:22:34.265383407Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"c450350043fa44aafd634b6cc5a455466b15ff4dfaa98d3662ad293961526476","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114994824758169600","occurred_at":"2026-10-27T08:17:15Z","failed_at":"e2e","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"f6f5d5c48156dc7017f642fad76beb2d815d485e2e13f47eac53e6bf0e430a2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114994824758169600","occurred_at":"2026-10-27T08:17:15Z","failed_at":"e2e","signature_id":"00fdd034291919a7a10b2949bec9498d190584b612339286dbbc735668b40e48","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-2tvl78\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21f49b568820\u003e: \n    DNS for route host agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x21f49d39a3e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-2tvl78\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21f49b568820\u003e: \n    DNS for route host agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x21f49d39a3e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cr9l5.apps.aro.t2i0u7o1r9d6j8r.s85t.j8169600.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bvlxv.apps.aro.p6n9f5i0h7d1x2n.1mdy.j9327744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6714/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114993082914050048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6714,"pr_state":"open","pr_sha":"fe4ca03e5d60bee2875c114ab7540e221b96baa4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T08:10:20Z","started_at":"2026-10-27T08:10:20Z","completed_at":"2026-10-27T10:39:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T10:43:42.103738539Z","timing_metadata_checked_at":"2026-10-27T10:43:42.103738539Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T10:43:42.107148428Z","region_metadata_checked_at":"2026-10-27T10:43:42.107148428Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"933b37a9749c07038aa8d20ebabd16ab26c7a59127028d7046b72ab4bcc7d817","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6714/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114993082914050048","occurred_at":"2026-10-27T08:10:20Z","failed_at":"e2e","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114987666448584704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"a630f5d29ba754ca53a2c8994a802571baafa9d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T07:48:48Z","started_at":"2026-10-27T07:48:48Z","completed_at":"2026-10-27T09:51:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T10:01:58.27477614Z","timing_metadata_checked_at":"2026-10-27T10:01:58.27477614Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T10:01:58.279366025Z","region_metadata_checked_at":"2026-10-27T10:01:58.279366025Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5ffe9d12e5ac61f7345d04a093cbe767c1b133a120aec23c32bb3c847a2e9959","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114987666448584704","occurred_at":"2026-10-27T07:48:48Z","failed_at":"e2e","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982798447087616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"0b953df1a904c8ed489fe9bb6cac1c0f154a9c94","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T07:29:28Z","started_at":"2026-10-27T07:29:28Z","completed_at":"2026-10-27T09:41:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:51:58.635859527Z","timing_metadata_checked_at":"2026-10-27T09:51:58.635859527Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:51:58.639340116Z","region_metadata_checked_at":"2026-10-27T09:51:58.639340116Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2c50b7043a624d7747a9e442a54153127f56179f0be1ed57cb58bc2b23a3b099","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982798447087616","occurred_at":"2026-10-27T07:29:28Z","failed_at":"e2e","signature_id":"dce5e5ce5976358e3c7a925a1fbd158f7e68fc614b9e70512494514a648c8b2a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"a13ec985e894fe5ddbd3419beeae247e34634a6438c53a5983bd19ab41eea4e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982798447087616","occurred_at":"2026-10-27T07:29:28Z","failed_at":"e2e","signature_id":"7e879e25f61ce940e144bedff05c8b377c887bfe979c26ff189023f3cf18a17c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2555ba2ee820\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x2555ba2ee800\u003e{\n            observation: \u003c*errors.errorString | 0x2555b9ce7460\u003e{\n                s: \"dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2555ba2ee7e0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2555b95a8370\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2555ba2ee7c0\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2555baea86f0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2555ba2ee820\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x2555ba2ee800\u003e{\n            observation: \u003c*errors.errorString | 0x2555b9ce7460\u003e{\n                s: \"dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2555ba2ee7e0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2555b95a8370\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2555ba2ee7c0\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2555baea86f0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4c7f20d280537e26756f2d14af08d1d2b02d836200b57c788ddc44687fa7d058","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","4c7f20d280537e26756f2d14af08d1d2b02d836200b57c788ddc44687fa7d058"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982484201443328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"closed","pr_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","final_merged_sha":"4cf164ecff36585c6e67a93232af75a00a26bf26","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T07:28:11Z","started_at":"2026-10-27T07:28:11Z","completed_at":"2026-10-27T09:41:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:51:58.517561111Z","timing_metadata_checked_at":"2026-10-27T09:51:58.517561111Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:51:58.521126499Z","region_metadata_checked_at":"2026-10-27T09:51:58.521126499Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"252f804287399f44df6f27c32c33c47e660f248df8b7a392d0b46053ddebdca9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982484201443328","occurred_at":"2026-10-27T07:28:11Z","failed_at":"e2e","signature_id":"bac0e3e75226f5af3bd5b014a630521a1cfaa2655fa7d3ded74b5c1381f24c77","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.003s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.003s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7761/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114977910874968064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7761,"pr_state":"closed","pr_sha":"55c5667b19edf70588255af6ff519fedfcc8ed78","final_merged_sha":"55c5667b19edf70588255af6ff519fedfcc8ed78","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T07:10:02Z","started_at":"2026-10-27T07:10:02Z","completed_at":"2026-10-27T09:13:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:19:29.426996229Z","timing_metadata_checked_at":"2026-10-27T09:19:29.426996229Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:19:29.437833994Z","region_metadata_checked_at":"2026-10-27T09:19:29.437833994Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4cbacce72239773530e17da622e1ba73bd0f23eb80ac3a88643dcf524fabc29f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7761/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114977910874968064","occurred_at":"2026-10-27T07:10:02Z","failed_at":"e2e","signature_id":"6a821c50d92310b73e1cf552308326b196dd980765b78ba09ceed32b40ce48d3","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.010s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.010s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114977151651418112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7770,"pr_state":"closed","pr_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","final_merged_sha":"ef1e3aa8841f8ca2a4c6bb6ee49707252c879255","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T07:07:01Z","started_at":"2026-10-27T07:07:01Z","completed_at":"2026-10-27T09:08:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:19:30.47735363Z","timing_metadata_checked_at":"2026-10-27T09:19:30.47735363Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:19:30.483774409Z","region_metadata_checked_at":"2026-10-27T09:19:30.483774409Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7045f34ef0d53f1ab2968a444fd4568c01854231e7a248f5f130d72ec874ec39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114977151651418112","occurred_at":"2026-10-27T07:07:01Z","failed_at":"e2e","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975739370541056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7769,"pr_state":"closed","pr_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","final_merged_sha":"b67a95a14236655cca9ae7dbad11b923f59d04d9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T07:01:08Z","started_at":"2026-10-27T07:01:08Z","completed_at":"2026-10-27T09:11:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:19:30.105000335Z","timing_metadata_checked_at":"2026-10-27T09:19:30.105000335Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:19:30.115329001Z","region_metadata_checked_at":"2026-10-27T09:19:30.115329001Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"eb0bfe91eca5c26a92a8df5c28f94f8d212a922fe91096a18425f6761956370d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975739370541056","occurred_at":"2026-10-27T07:01:08Z","failed_at":"e2e","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975256258023424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7768,"pr_state":"closed","pr_sha":"169563406cb32ad3707f97b94c7a229632cbe344","final_merged_sha":"169563406cb32ad3707f97b94c7a229632cbe344","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T06:59:30Z","started_at":"2026-10-27T06:59:30Z","completed_at":"2026-10-27T09:17:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:19:29.784228273Z","timing_metadata_checked_at":"2026-10-27T09:19:29.784228273Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:19:29.788588159Z","region_metadata_checked_at":"2026-10-27T09:19:29.788588159Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"00c620550e09d409c1dabf14d7407fee643e875815d72685cdc82ee7a8c5a4d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975256258023424","occurred_at":"2026-10-27T06:59:30Z","failed_at":"e2e","signature_id":"17d4cdfc77354f622fc0109e20b904a9def3d63396c30a68d9e99bbbfc413481","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"2872dee03c5e8214b18e295f761b4c4b6fb5e08a3b87d22ce88a973603f6b541","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975256258023424","occurred_at":"2026-10-27T06:59:30Z","failed_at":"e2e","signature_id":"fd1c1e45c21971c8bd7dc5c6ce80f975b1981c85f1832b8a925316f35d943923","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19f0aa4bea20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-wj8x9hp7bqx5\" to finish creating: GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"name\": \"6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:09:38.679552409Z\",\n      \"endTime\": \"2026-10-27T08:28:40.674204457Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n            correlation ID: 51afa422-8d59-4334-91a3-bc90ade1f885\n            client request ID: ebb6ee5a-68e9-481e-8fab-49a3ebe79568\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19f0aa4bea20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-wj8x9hp7bqx5\" to finish creating: GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"name\": \"6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:09:38.679552409Z\",\n      \"endTime\": \"2026-10-27T08:28:40.674204457Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n            correlation ID: 51afa422-8d59-4334-91a3-bc90ade1f885\n            client request ID: ebb6ee5a-68e9-481e-8fab-49a3ebe79568\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"5bbe82813cee485ba01f3d4a450f0a6b53de7593","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T06:56:08Z","started_at":"2026-10-27T06:56:08Z","completed_at":"2026-10-27T09:04:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T09:10:36.829690752Z","timing_metadata_checked_at":"2026-10-27T09:10:36.829690752Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T09:10:36.833809538Z","region_metadata_checked_at":"2026-10-27T09:10:36.833809538Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"48980ad9a783a9bf0f9f7b0dcf419ae80d7af19165ff94fa6f2a0f2cd452f7d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","failed_at":"e2e","signature_id":"f56a1a7fe0529dc820427621b1f6eaead9f3912f807ee59dc2e3a11a74974b94","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x211f21252720\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x211f21252720\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4d661fb661180412d79fc99146fbf8d02151376731f8cbdb584bccc883aaf3e0","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0"}},{"row_id":"712900778c2991ba1d8f7dd45519919892ee3bea9947d6846d5444168537f33b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","failed_at":"alert","signature_id":"33415ae06c670660cd19c33a16e1db7f18b2fe42419fc9b5576f8033ad4489ab","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T08:03:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1218048-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.120:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1218048/providers/microsoft.monitor/accounts/services-j1218048\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-487l5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1f3e87ec-25dc-43bc-b62e-8a9a539fc230\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-487l5 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T08:06:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1218048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1218048/providers/microsoft.monitor/accounts/services-j1218048\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-qrn9k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0c2ca63-da52-4110-a38b-c0b2caede123\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-qrn9k (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T08:06:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1218048-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.120:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1218048/providers/microsoft.monitor/accounts/services-j1218048\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-487l5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1f3e87ec-25dc-43bc-b62e-8a9a539fc230\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-487l5 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T08:06:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1218048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1218048/providers/microsoft.monitor/accounts/services-j1218048\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-qrn9k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0c2ca63-da52-4110-a38b-c0b2caede123\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-qrn9k (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"d4db93d255462592fe254733739534ef741ab571f0354fb088e4fc20d53c0330","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","failed_at":"e2e","signature_id":"54722d0e8af7431db3f910c6b8799d624cfe0c8f4b5fa53109f9ee0b900c8d0d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-458kh7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1b0cdceb6450\u003e: \n    VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-qg48vc\" not yet present in zone rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-qg48vc' does not exist in resource group 'rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n            correlation ID: 1332f9fe-dcac-4684-bfa0-1d5d13e9cf06\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-458kh7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1b0cdceb6450\u003e: \n    VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-qg48vc\" not yet present in zone rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-qg48vc' does not exist in resource group 'rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n            correlation ID: 1332f9fe-dcac-4684-bfa0-1d5d13e9cf06\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a82463256e75ac51402f90e1a076775c96cdeed8daf25e578b182ea9407bbbf8","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-qg48vc' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}},{"row_id":"f7ca8286bddf6c6b2b594508f4709f47bd613eeb26c5ce8caef23310e4d1c1fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","failed_at":"e2e","signature_id":"13be5e1341477c52eade788cb598f1cc4e1ab24ef4f65e65ff9ddcdba8af23bb","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.006s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.006s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","4d661fb661180412d79fc99146fbf8d02151376731f8cbdb584bccc883aaf3e0","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","a82463256e75ac51402f90e1a076775c96cdeed8daf25e578b182ea9407bbbf8"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114973576971948032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"2ef2d62beec7d2455990de0db7018596770b4035","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T06:52:49Z","started_at":"2026-10-27T06:52:49Z","completed_at":"2026-10-27T07:38:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T07:47:34.770007414Z","timing_metadata_checked_at":"2026-10-27T07:47:34.770007414Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T07:47:34.773607102Z","region_metadata_checked_at":"2026-10-27T07:47:34.773607102Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"aa12df4ff85273c4e36ed1874f3dbd61a6048b4633ae4340e602f808405a054f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114973576971948032","occurred_at":"2026-10-27T06:52:49Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-27T07:31:38.140Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-27T07:31:38.545Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-27T07:31:39.254Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1948032-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0\ntime=2026-10-27T07:33:09.610Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1948032-mgmt-1/providers/Microsoft.Resources/deployments/95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0/operationStatuses/08584085205866172617\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1948032-mgmt-1/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"InvalidRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Call to Microsoft.EventGrid/namespaces failed. Error message: Required property 'linkIdentifier' not found in JSON. Path 'remotePrivateEndpoint.connectionDetails[0]', line 1, position 932.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": []\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8199ed0284e70ecfb2dc70637f49d2db3d9241a37d6a24ae7dcc8841eac024f4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidRequest; detail message Call to Microsoft.EventGrid/namespaces failed. Error message: Required property 'linkIdentifier' not found in JSON. Path 'remotePrivateEndpoint.connectionDetails[0]', line 1, position 932.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8199ed0284e70ecfb2dc70637f49d2db3d9241a37d6a24ae7dcc8841eac024f4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114964013585534976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T06:14:49Z","started_at":"2026-10-27T06:14:49Z","completed_at":"2026-10-27T08:21:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T08:29:21.553489545Z","timing_metadata_checked_at":"2026-10-27T08:29:21.553489545Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T08:29:21.557537732Z","region_metadata_checked_at":"2026-10-27T08:29:21.557537732Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"341594f5a521ba846acb559075d918c1d7c4c2533d9a2f0adf854aa41728055c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114964013585534976","occurred_at":"2026-10-27T06:14:49Z","failed_at":"e2e","signature_id":"6451a4f5c64d9abbc32284dc078828b1f9e659530fba80fcbdd390788556e7fa","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"45b490f8f8c6a73eb5cf3fc76ae70b1654bd53894cc9ca7dae67ec1077ddc3f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114964013585534976","occurred_at":"2026-10-27T06:14:49Z","failed_at":"alert","signature_id":"ad318a0839ef635cdd7da0dbb7b359b7c65595221514e368750bb1954fa66624","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 30 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-wc6lw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-wc6lw has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-p9hg8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-p9hg8 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9b7ql\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-9b7ql has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-lc889\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-lc889 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-x86ml\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-x86ml has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-4xt6q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-4xt6q has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-h6d99\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-h6d99 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-vnjz9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-vnjz9 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7wmqn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-7wmqn has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ns7vc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-ns7vc has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-swpfn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-swpfn has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-gjxvj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-gjxvj has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-22wkx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-22wkx has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-nznss\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-nznss has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-b6thp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-b6thp has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-78krz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-78krz has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-s4wx7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-s4wx7 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-58z5p\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-58z5p has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-kkjkc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-kkjkc has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-t9xnm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-t9xnm has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-29668\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-29668 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8j499\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-8j499 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-27kpw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-27kpw has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-2vj4q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-2vj4q has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-wkflh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-wkflh has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ct72w\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-ct72w has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-jwkmm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-jwkmm has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-d7q62\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-d7q62 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-4g9hz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-4g9hz has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T07:02:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5534976-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-56n52\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-67995d8d8b-56n52 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114952950173405184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7740,"pr_state":"closed","pr_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","final_merged_sha":"c26c6b97becc4dc84e6b25718f340f7a6d0d2d74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T05:30:51Z","started_at":"2026-10-27T05:30:51Z","completed_at":"2026-10-27T07:46:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T07:58:17.857059385Z","timing_metadata_checked_at":"2026-10-27T07:58:17.857059385Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T07:58:17.861420871Z","region_metadata_checked_at":"2026-10-27T07:58:17.861420871Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b3d5a3ffb49d2c2ffa93f2aa7f7c5dd9477897e02dc761db51b07a313e2a60fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114952950173405184","occurred_at":"2026-10-27T05:30:51Z","failed_at":"e2e","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7767,"pr_state":"open","pr_sha":"003fbd9857bd03e441a2d6acb943c5e2fd77daea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T03:30:25Z","started_at":"2026-10-27T03:30:25Z","completed_at":"2026-10-27T06:05:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T06:12:45.511746978Z","timing_metadata_checked_at":"2026-10-27T06:12:45.511746978Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T06:12:45.515660066Z","region_metadata_checked_at":"2026-10-27T06:12:45.515660066Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5de2549fcd838460bc4f5fb0ac682337cb2cc942f0d8e777d15b2d19addc8d0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","occurred_at":"2026-10-27T03:30:25Z","failed_at":"e2e","signature_id":"9695fdc7ed44a5281f86995428490bd94b9fe5e26b6dc18c50bb5151a4ed31bb","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x3462d072440?, {0x7f705437c4e0?, 0x3462db47650?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1197 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x3462df2c480, {0x7f705437c4e0, 0x3462db47650}, {0xfd7ebbf, 0xa}, {0x3462cfcc129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func27.2.1({0x7f705437c4e0, 0x3462db47650})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:141 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x105c3910?, 0x3462db47650?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 80\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x3462d072440?, {0x7f705437c4e0?, 0x3462db47650?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1197 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x3462df2c480, {0x7f705437c4e0, 0x3462db47650}, {0xfd7ebbf, 0xa}, {0x3462cfcc129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func27.2.1({0x7f705437c4e0, 0x3462db47650})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:141 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x105c3910?, 0x3462db47650?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 80\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret"}},{"row_id":"988934cac1e9037b6dc81f264bb99948dca4a1f34b81668267ae2f62307b05d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","occurred_at":"2026-10-27T03:30:25Z","failed_at":"e2e","signature_id":"dce5e5ce5976358e3c7a925a1fbd158f7e68fc614b9e70512494514a648c8b2a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"bf4716b9992d14147f53c05db57a6579dac43b3eddd6eb73d6acd7cdc9b679de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","occurred_at":"2026-10-27T03:30:25Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T05:07:12Z\n  Ended:       2026-10-27T05:24:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8907392-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"2ef2d62beec7d2455990de0db7018596770b4035","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T03:15:51Z","started_at":"2026-10-27T03:15:51Z","completed_at":"2026-10-27T05:19:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T05:30:00.140094894Z","timing_metadata_checked_at":"2026-10-27T05:30:00.140094894Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T05:30:00.146049274Z","region_metadata_checked_at":"2026-10-27T05:30:00.146049274Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"08599e761ef186dd077fa605abad4379c8b7b63b6c650a230a4aaee18388e7ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","occurred_at":"2026-10-27T03:15:51Z","failed_at":"e2e","signature_id":"61541f3b5ca8921faa4b85328182f5e428d4643d60841d69d788f32ca5efe589","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-jz7tkx\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3490511d0108\u003e: \n    VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) timed out after 11m45.097s: DNS A record \"*.e2e-dns-4sk7fq\" not yet present in zone rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-4sk7fq' does not exist in resource group 'rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n            correlation ID: 7140a367-bc3d-43f2-b136-ffe5167138f0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-jz7tkx\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3490511d0108\u003e: \n    VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) timed out after 11m45.097s: DNS A record \"*.e2e-dns-4sk7fq\" not yet present in zone rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-4sk7fq' does not exist in resource group 'rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n            correlation ID: 7140a367-bc3d-43f2-b136-ffe5167138f0\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f507da136c9daa16f6c852638ab5c837615df7fc7163e85f3926a57d8414cb44","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-4sk7fq' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound"}},{"row_id":"759d67ce91d96a07ff56471d8a2d6d810e21018fc45099fd5aa88104f2e0835f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","occurred_at":"2026-10-27T03:15:51Z","failed_at":"e2e","signature_id":"afd200cebfca006fbbf28cf7ff8dfd7883685e35a9e278667284486e031f242f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2680bac6c1a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2680bac6c180\u003e{\n            observation: \u003c*errors.errorString | 0x2680b95b3860\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2680bac6c160\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2680b9c4c190\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2680bac6c140\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2680bb4eaab0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2680bac6c1a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2680bac6c180\u003e{\n            observation: \u003c*errors.errorString | 0x2680b95b3860\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2680bac6c160\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2680b9c4c190\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2680bac6c140\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2680bb4eaab0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"818794cda20cc99c353a743beee80b93b7a1886f28dfb74144dded0f0d86e651","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}},{"row_id":"90490d68964335381c323c2d2f8b3b932ec328daf38951dd6d411070301e2d66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","occurred_at":"2026-10-27T03:15:51Z","failed_at":"e2e","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","818794cda20cc99c353a743beee80b93b7a1886f28dfb74144dded0f0d86e651","f507da136c9daa16f6c852638ab5c837615df7fc7163e85f3926a57d8414cb44"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918518389477376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T03:14:02Z","started_at":"2026-10-27T03:14:02Z","completed_at":"2026-10-27T05:09:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T05:19:24.512267957Z","timing_metadata_checked_at":"2026-10-27T05:19:24.512267957Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T05:19:24.516529243Z","region_metadata_checked_at":"2026-10-27T05:19:24.516529243Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3ab1f7ae65909465b78068375bdfd2f9390dec9b0f560ade0da98fc345e70c8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918518389477376","occurred_at":"2026-10-27T03:14:02Z","failed_at":"e2e","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T01:13:24Z","started_at":"2026-10-27T01:13:24Z","completed_at":"2026-10-27T03:12:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T03:22:43.74100777Z","timing_metadata_checked_at":"2026-10-27T03:22:43.74100777Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T03:22:43.743944561Z","region_metadata_checked_at":"2026-10-27T03:22:43.743944561Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"57d8f7770eb6fef64e6afe4e9de29cfc83bef5bc54b2ba8e7b44379f64f801cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","failed_at":"alert","signature_id":"7daeb4e2c89121c0d6a0276aae92631419d4fb62396974536d0feb3f60deb191","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T02:11:40Z\n  Ended:       2026-10-27T02:36:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1152000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1152000/providers/microsoft.monitor/accounts/hcps-j1152000\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-zp65c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T02:11:40Z\n  Ended:       2026-10-27T02:37:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1152000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1152000/providers/microsoft.monitor/accounts/hcps-j1152000\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-zp65c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"6ad51356fac2d3e6711332f77aa8ff3f4500b7d5fd832feee0043c6b53883fdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","failed_at":"alert","signature_id":"482db436f84040b93006731be0326af709da86d17e4d0ef8870af1a8f0485a6b","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T02:25:40Z\n  Ended:       2026-10-27T02:37:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1152000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1152000/providers/microsoft.monitor/accounts/hcps-j1152000\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-zp65c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T02:26:36Z\n  Ended:       2026-10-27T02:37:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1152000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1152000/providers/microsoft.monitor/accounts/hcps-j1152000\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-zp65c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"7aecd6232a3613663deb91c661f57168ee861f6a988322a9edf1e273e16672c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","failed_at":"e2e","signature_id":"847c23767aa51d34dbcbd44bfae866c902845ee3aca8c91c84f1387745ed29c9","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x250d4d0585c0\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-8dj5fcfzk684\" to finish creating: GET https://rp.j1152000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5ecd395-85d4-4167-8c0b-d86ab4676770\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5ecd395-85d4-4167-8c0b-d86ab4676770\",\n      \"name\": \"d5ecd395-85d4-4167-8c0b-d86ab4676770\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T01:51:23.29970675Z\",\n      \"endTime\": \"2026-10-27T02:20:31.588657474Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1152000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5ecd395-85d4-4167-8c0b-d86ab4676770\n            correlation ID: b2ed5fd1-aced-406c-9da8-d4a42aa70231\n            client request ID: 2be74574-9856-48bd-8976-fff7fa4252d8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x250d4d0585c0\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-8dj5fcfzk684\" to finish creating: GET https://rp.j1152000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5ecd395-85d4-4167-8c0b-d86ab4676770\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5ecd395-85d4-4167-8c0b-d86ab4676770\",\n      \"name\": \"d5ecd395-85d4-4167-8c0b-d86ab4676770\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T01:51:23.29970675Z\",\n      \"endTime\": \"2026-10-27T02:20:31.588657474Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1152000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5ecd395-85d4-4167-8c0b-d86ab4676770\n            correlation ID: b2ed5fd1-aced-406c-9da8-d4a42aa70231\n            client request ID: 2be74574-9856-48bd-8976-fff7fa4252d8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"82ced8deedb65b112047cc116d25de03ae5c62af1d3d4f296d3908a6cd6f2354","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T02:08:12Z\n  Ended:       2026-10-27T02:18:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1152000-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T02:08:12Z\n  Ended:       2026-10-27T02:18:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1152000-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"989b6e1b35b6fdaa86b1f169b34641ca005583c7bf50f2f1e2168b1df5f6e150","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","failed_at":"alert","signature_id":"090543014f12d2faca82ee316f5e8a3161876854e37b7b7810c6f5097d0e8fb6","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T02:13:36Z\n  Ended:       2026-10-27T02:41:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1152000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1152000/providers/microsoft.monitor/accounts/hcps-j1152000\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0cbd6fe7-cf8c-4402-9cca-95a6f5bddfe0\"\n  Description: Pod ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-27T02:14:36Z\n  Ended:       2026-10-27T02:41:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1152000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.174:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1152000/providers/microsoft.monitor/accounts/hcps-j1152000\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0cbd6fe7-cf8c-4402-9cca-95a6f5bddfe0\"\n  Description: Pod ocm-arohcpci01-2tmrb70fk8q8hg0qepkmrjljph9ejlcl-ksm-hcp-metrics/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"a5ca95e70c5f3156938fe507775842cd5806f29b5a028362ecb296dbeb41debb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","failed_at":"e2e","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114886591380983808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7739,"pr_state":"closed","pr_sha":"a38be30ac65f8313cef5f9fefaca1721c2f80784","final_merged_sha":"85f478eff30207b95d439e1d26791f75c7ff20f7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T01:07:10Z","started_at":"2026-10-27T01:07:10Z","completed_at":"2026-10-27T03:22:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T03:32:51.261963326Z","timing_metadata_checked_at":"2026-10-27T03:32:51.261963326Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T03:32:51.265883813Z","region_metadata_checked_at":"2026-10-27T03:32:51.265883813Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"29060ddd361895982b88bf51d4877065abcec98a6070f225cd434eef190201e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114886591380983808","occurred_at":"2026-10-27T01:07:10Z","failed_at":"e2e","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114870474402959360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"closed","pr_sha":"27fe8fe46cb154c4b4ff583dea5a4d9ad0f93dd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-27T00:03:08Z","started_at":"2026-10-27T00:03:08Z","completed_at":"2026-10-27T02:34:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T02:50:52.801543493Z","timing_metadata_checked_at":"2026-10-27T02:50:52.801543493Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T02:50:52.805858678Z","region_metadata_checked_at":"2026-10-27T02:50:52.805858678Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1f24be464b39b6827bb35fdaef929bf11d2b2b34e0d90b3cfee20403c2c2f8a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114870474402959360","occurred_at":"2026-10-27T00:03:08Z","failed_at":"e2e","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"74a10ecbe270eb593bf8ff53b02aa8a7492195f16bb96698722e60846c7638bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114870474402959360","occurred_at":"2026-10-27T00:03:08Z","failed_at":"e2e","signature_id":"171fe6317ac5031415995a5b49fd173b8d76345e4653ed4a832f67862f6d0e2b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26678ea60858\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f8mftsvbkmbp\" to finish creating: GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"name\": \"95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T01:20:09.676863434Z\",\n      \"endTime\": \"2026-10-27T01:39:10.703576848Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n            correlation ID: 7f67a06a-5085-4c8e-9661-1df8e8a79a0a\n            client request ID: 9930c3d6-bc48-4ffb-b0a4-b32c7c60a790\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26678ea60858\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f8mftsvbkmbp\" to finish creating: GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"name\": \"95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T01:20:09.676863434Z\",\n      \"endTime\": \"2026-10-27T01:39:10.703576848Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n            correlation ID: 7f67a06a-5085-4c8e-9661-1df8e8a79a0a\n            client request ID: 9930c3d6-bc48-4ffb-b0a4-b32c7c60a790\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea06060b15ab7907c4f534ea9c38687a98d66cbc48379a8dd476993e78886883","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","ea06060b15ab7907c4f534ea9c38687a98d66cbc48379a8dd476993e78886883"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]}]}]}