{"meta":{"date":"2026-10-10","timezone":"UTC","generated_at":"2026-11-01T09:19:53Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":14,"failed_runs":7,"runs_with_occurrences":7,"runs_with_failure_pattern_matches":7,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":7,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108936045339873280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7374,"pr_state":"closed","pr_sha":"2654cb9b188ba5ee0ce25abebb6c7c5a46bc1f47","final_merged_sha":"2654cb9b188ba5ee0ce25abebb6c7c5a46bc1f47","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-10T15:01:49Z","started_at":"2026-10-10T15:01:49Z","completed_at":"2026-10-10T17:06:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T17:17:01.028731905Z","timing_metadata_checked_at":"2026-10-10T17:17:01.028731905Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T17:17:01.03341489Z","region_metadata_checked_at":"2026-10-10T17:17:01.03341489Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7456/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108911869879128064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7456,"pr_state":"closed","pr_sha":"de99ea9212ca8ef13484354c7f4310408b98a515","final_merged_sha":"de99ea9212ca8ef13484354c7f4310408b98a515","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-10T13:25:46Z","started_at":"2026-10-10T13:25:46Z","completed_at":"2026-10-10T15:43:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T15:51:56.622103595Z","timing_metadata_checked_at":"2026-10-10T15:51:56.622103595Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T15:51:56.627642679Z","region_metadata_checked_at":"2026-10-10T15:51:56.627642679Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108890869657505792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"closed","pr_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","final_merged_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-10T12:02:19Z","started_at":"2026-10-10T12:02:19Z","completed_at":"2026-10-10T14:51:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T14:59:02.168084374Z","timing_metadata_checked_at":"2026-10-10T14:59:02.168084374Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T14:59:02.17247046Z","region_metadata_checked_at":"2026-10-10T14:59:02.17247046Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108886756110635008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7374,"pr_state":"closed","pr_sha":"2654cb9b188ba5ee0ce25abebb6c7c5a46bc1f47","final_merged_sha":"2654cb9b188ba5ee0ce25abebb6c7c5a46bc1f47","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-10T11:45:56Z","started_at":"2026-10-10T11:45:56Z","completed_at":"2026-10-10T14:06:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T14:17:45.799737913Z","timing_metadata_checked_at":"2026-10-10T14:17:45.799737913Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T14:17:45.8037319Z","region_metadata_checked_at":"2026-10-10T14:17:45.8037319Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"26b323105ecd038ecfdd7756c0c93b5772af4cd2323a21c4a617d85d528e4234","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108886756110635008","occurred_at":"2026-10-10T11:45:56Z","failed_at":"e2e","signature_id":"9df37f22bf11ea31242e52f951190e90a4071cb737774ac577107261d4da179e","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2179351f8800\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-5d4jlxd268w8\" to finish creating: GET https://rp.j0635008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\",\n      \"name\": \"5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T12:54:39.340662434Z\",\n      \"endTime\": \"2026-10-10T12:55:18.027112085Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-5d4jlxd268w8/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0635008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\n            correlation ID: 13bc1f0a-5221-4912-b516-aa03e8a99e61\n            client request ID: 5d284363-4eac-4367-9c73-8148dde5d509\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2179351f8800\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-5d4jlxd268w8\" to finish creating: GET https://rp.j0635008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\",\n      \"name\": \"5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T12:54:39.340662434Z\",\n      \"endTime\": \"2026-10-10T12:55:18.027112085Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-5d4jlxd268w8/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0635008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a75eacc-f787-4eeb-91c6-ba37f2f3c7e0\n            correlation ID: 13bc1f0a-5221-4912-b516-aa03e8a99e61\n            client request ID: 5d284363-4eac-4367-9c73-8148dde5d509\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c842691ee916dd7172d0c239a927f62c93874f7f5f2540ccd5e604642687c1b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108886756110635008","occurred_at":"2026-10-10T11:45:56Z","failed_at":"alert","signature_id":"569ddb71f13f546121c7971b77e866f8e636b91dfed46d5b29bdb27c60834dd7","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-10T13:49:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j0635008-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.197:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0635008/providers/microsoft.monitor/accounts/services-j0635008\", namespace=\"klusterlet-2tbu9f66u76ktrmgh418ji9vp03p5kjp\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8w74x\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tbu9f66u76ktrmgh418ji9vp03p5kjp on cluster ci01-j0635008-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7456/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108841512396656640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7456,"pr_state":"closed","pr_sha":"de99ea9212ca8ef13484354c7f4310408b98a515","final_merged_sha":"de99ea9212ca8ef13484354c7f4310408b98a515","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-10T08:46:11Z","started_at":"2026-10-10T08:46:11Z","completed_at":"2026-10-10T10:54:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T11:11:13.69300319Z","timing_metadata_checked_at":"2026-10-10T11:11:13.69300319Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T11:11:13.69650058Z","region_metadata_checked_at":"2026-10-10T11:11:13.69650058Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108830741738557440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"closed","pr_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","final_merged_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-10T08:03:23Z","started_at":"2026-10-10T08:03:23Z","completed_at":"2026-10-10T10:21:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T10:39:38.834042028Z","timing_metadata_checked_at":"2026-10-10T10:39:38.834042028Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T10:39:38.839530211Z","region_metadata_checked_at":"2026-10-10T10:39:38.839530211Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"37a6a4ca0eb1dd4a9d65b061f9dd23a401455251d2e155ff22513bd8af855097","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108830741738557440","occurred_at":"2026-10-10T08:03:23Z","failed_at":"e2e","signature_id":"d2c8e054823fbd376c984214c001bc93b2a900e7a4a75bde428fa6fea3d24178","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5b5e959eb80\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hxz5lhgx6pgc\" to finish creating: GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"name\": \"bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T09:08:14.887674935Z\",\n      \"endTime\": \"2026-10-10T09:37:18.731959559Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n            correlation ID: 85344fe9-db3b-408b-ba74-a3aed4f6938b\n            client request ID: 900c9a16-ab81-46bf-aba5-3f67f267761a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5b5e959eb80\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hxz5lhgx6pgc\" to finish creating: GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"name\": \"bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T09:08:14.887674935Z\",\n      \"endTime\": \"2026-10-10T09:37:18.731959559Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n            correlation ID: 85344fe9-db3b-408b-ba74-a3aed4f6938b\n            client request ID: 900c9a16-ab81-46bf-aba5-3f67f267761a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108819747389837312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7374,"pr_state":"closed","pr_sha":"2654cb9b188ba5ee0ce25abebb6c7c5a46bc1f47","final_merged_sha":"2654cb9b188ba5ee0ce25abebb6c7c5a46bc1f47","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-10T07:19:42Z","started_at":"2026-10-10T07:19:42Z","completed_at":"2026-10-10T09:46:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T09:57:37.935500676Z","timing_metadata_checked_at":"2026-10-10T09:57:37.935500676Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T09:57:37.938481766Z","region_metadata_checked_at":"2026-10-10T09:57:37.938481766Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2f3b271bd915438269edd76490716bd51de11f547992deb91f4de9649f9391b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108819747389837312","occurred_at":"2026-10-10T07:19:42Z","failed_at":"alert","signature_id":"c86eaf93a1267e5b8897c354444574607bb8dab7fce27496ea0d79db2502e9d3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-10T08:50:24Z\n  Ended:       2026-10-10T08:56:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9837312-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbqdq7nkko6nfm8vch1p3aq46onal67-private-kas\", pod=\"cluster-policy-controller-86bf79cf56-rh2l8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tbqdq7nkko6nfm8vch1p3aq46onal67-private-kas/cluster-policy-controller-86bf79cf56-rh2l8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108815895240380416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"closed","pr_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","final_merged_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-10T07:04:23Z","started_at":"2026-10-10T07:04:23Z","completed_at":"2026-10-10T09:43:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T09:57:37.453313116Z","timing_metadata_checked_at":"2026-10-10T09:57:37.453313116Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T09:57:37.457162804Z","region_metadata_checked_at":"2026-10-10T09:57:37.457162804Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108810609872080896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"ff34e728fd78c7cec4e7c78f5ec6799884f45c36","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-10T06:43:23Z","started_at":"2026-10-10T06:43:23Z","completed_at":"2026-10-10T09:00:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T09:15:45.595816916Z","timing_metadata_checked_at":"2026-10-10T09:15:45.595816916Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T09:15:45.601453299Z","region_metadata_checked_at":"2026-10-10T09:15:45.601453299Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108804931556216832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"closed","pr_sha":"982cceaae574f524cd7649b27b4601cf50463486","final_merged_sha":"09587f6f09c91b53ea705711a675d523def6cece","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-10T06:20:50Z","started_at":"2026-10-10T06:20:50Z","completed_at":"2026-10-10T08:46:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T09:04:48.138419885Z","timing_metadata_checked_at":"2026-10-10T09:04:48.138419885Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T09:04:48.142190874Z","region_metadata_checked_at":"2026-10-10T09:04:48.142190874Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108785286350639104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"closed","pr_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","final_merged_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-10T05:02:46Z","started_at":"2026-10-10T05:02:46Z","completed_at":"2026-10-10T07:24:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T07:41:25.304899776Z","timing_metadata_checked_at":"2026-10-10T07:41:25.304899776Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T07:41:25.310432959Z","region_metadata_checked_at":"2026-10-10T07:41:25.310432959Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7f9d5cb53be13ffacb36f39b4b1553af5aa8805b93cf4c9843572b3ef95a45e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108785286350639104","occurred_at":"2026-10-10T05:02:46Z","failed_at":"e2e","signature_id":"5ed4b18dce6b054ef58ec3f72cbaa45d685da5b7aa78958846e995ebb0a58680","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24a46d574920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlxq4qgj7xpp\" to finish creating: GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\",\n      \"name\": \"309223c5-c91d-4270-b774-50dec66e2570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T06:15:46.054292016Z\",\n      \"endTime\": \"2026-10-10T06:44:53.542490042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-drwbc-mbw6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-drwbc-mbw6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n            correlation ID: 57417513-291c-4d90-a0bc-a093da0940a4\n            client request ID: 3e97b4e2-ed3b-4f11-be4f-69ec5d075229\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24a46d574920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlxq4qgj7xpp\" to finish creating: GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\",\n      \"name\": \"309223c5-c91d-4270-b774-50dec66e2570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T06:15:46.054292016Z\",\n      \"endTime\": \"2026-10-10T06:44:53.542490042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-drwbc-mbw6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-drwbc-mbw6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n            correlation ID: 57417513-291c-4d90-a0bc-a093da0940a4\n            client request ID: 3e97b4e2-ed3b-4f11-be4f-69ec5d075229\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"03ddc6ab96e7bba4116d23faa9db196492be0ac8bee0a6170184bf1e8ecbc394","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["03ddc6ab96e7bba4116d23faa9db196492be0ac8bee0a6170184bf1e8ecbc394"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108739904140742656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-10T02:02:26Z","started_at":"2026-10-10T02:02:26Z","completed_at":"2026-10-10T04:29:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T04:41:18.14856982Z","timing_metadata_checked_at":"2026-10-10T04:41:18.14856982Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T04:41:18.152332409Z","region_metadata_checked_at":"2026-10-10T04:41:18.152332409Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7fe2944503ed5d26cfa92107e2f2df4ffd94caeba1e8d1909911508bb7564a64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108739904140742656","occurred_at":"2026-10-10T02:02:26Z","failed_at":"e2e","signature_id":"899ee92e21aa873413f2b225dc269879ee50d989eb61e324bf237f0a792412c9","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23a798e4e3a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7kr9hzf26ffd\" to finish creating: GET https://rp.j0742656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"name\": \"2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T03:18:06.524337179Z\",\n      \"endTime\": \"2026-10-10T03:47:07.577893958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23a798e4e3a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7kr9hzf26ffd\" to finish creating: GET https://rp.j0742656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"name\": \"2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T03:18:06.524337179Z\",\n      \"endTime\": \"2026-10-10T03:47:07.577893958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108729709905842176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"open","pr_sha":"868b002d86683930a48766871bcfcccf9bbeb999","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-10T01:21:53Z","started_at":"2026-10-10T01:21:53Z","completed_at":"2026-10-10T03:30:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T03:48:44.780256431Z","timing_metadata_checked_at":"2026-10-10T03:48:44.780256431Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T03:48:44.784217116Z","region_metadata_checked_at":"2026-10-10T03:48:44.784217116Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bdfe67b08c642f4c1c107a080db4ea6259ff141ad970f3cd173b181d9406fb59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108729709905842176","occurred_at":"2026-10-10T01:21:53Z","failed_at":"e2e","signature_id":"0b3ceb5461aa5f357bf0cb7f974c95380ac2a402e404e4401b031539454edcc4","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x560689fa080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jzlxwvqzg694\" to finish creating: GET https://rp.j5842176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"name\": \"efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T02:25:01.899620825Z\",\n      \"endTime\": \"2026-10-10T02:54:06.245842706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x560689fa080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jzlxwvqzg694\" to finish creating: GET https://rp.j5842176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"name\": \"efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T02:25:01.899620825Z\",\n      \"endTime\": \"2026-10-10T02:54:06.245842706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108710710912487424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"closed","pr_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","final_merged_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-10T00:06:26Z","started_at":"2026-10-10T00:06:26Z","completed_at":"2026-10-10T02:19:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T02:35:07.540715561Z","timing_metadata_checked_at":"2026-10-10T02:35:07.540715561Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T02:35:07.544102851Z","region_metadata_checked_at":"2026-10-10T02:35:07.544102851Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0e9b7c50447eca46e27146ceb3e9596546cff843993298ef34f76e53787fc8d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108710710912487424","occurred_at":"2026-10-10T00:06:26Z","failed_at":"e2e","signature_id":"86e4f5f8163a032f6e683ca40d132f158ebf85db8e38a520bc43b0ee11650f59","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x7f83ee01c00\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4ck5gxmvgl2\" to finish creating: GET https://rp.j2487424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b0e10a40-6968-4b3e-9cdc-0302b36aa6cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b0e10a40-6968-4b3e-9cdc-0302b36aa6cc\",\n      \"name\": \"b0e10a40-6968-4b3e-9cdc-0302b36aa6cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T01:21:08.093175992Z\",\n      \"endTime\": \"2026-10-10T01:50:13.893881459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c4ck5gxmvgl2--managed/providers/Microsoft.Compute/virtualMachines/m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c4ck5gxmvgl2--managed/providers/Microsoft.Compute/virtualMachines/m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x7f83ee01c00\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4ck5gxmvgl2\" to finish creating: GET https://rp.j2487424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b0e10a40-6968-4b3e-9cdc-0302b36aa6cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b0e10a40-6968-4b3e-9cdc-0302b36aa6cc\",\n      \"name\": \"b0e10a40-6968-4b3e-9cdc-0302b36aa6cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T01:21:08.093175992Z\",\n      \"endTime\": \"2026-10-10T01:50:13.893881459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c4ck5gxmvgl2--managed/providers/Microsoft.Compute/virtualMachines/m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c4ck5gxmvgl2--managed/providers/Microsoft.Compute/virtualMachines/m6u2s5a2u3w6s5j-np-delete-main-skm2q-qflrc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4ck5gxmvgl2\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}