{"meta":{"start_date":"2026-10-21","end_date":"2026-10-21","start_at":"2026-10-21T00:00:00Z","end_at":"2026-10-22T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T02:11:52Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":5,"matched_occurrences":5,"runs_affected":1},"rows":[{"environment":"int","failure_pattern_id":"8b157df38af3252928b0d0c8ebb12f675589f14b62d8751da69093589eb2fca4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c4c027a02e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6xshnph7ptd9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/363f69bc-d1f2-4956-842d-980aba640091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/363f69bc-d1f2-4956-842d-980aba640091\",\n      \"name\": \"363f69bc-d1f2-4956-842d-980aba640091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:02:55.100233745Z\",\n      \"endTime\": \"2026-10-21T21:21:58.935771192Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1s0i5w0t2h9v3h-np-1-br8d2-5c77c: Failed: virtualmachine failed to create or update. err: failed to get existing resource fips-enabled-6xshnph7ptd9--managed/j1s0i5w0t2h9v3h-np-1-br8d2-5c77c (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-6xshnph7ptd9--managed/providers/Microsoft.Compute/virtualMachines/j1s0i5w0t2h9v3h-np-1-br8d2-5c77c\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https:... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/363f69bc-d1f2-4956-842d-980aba640091\n            correlation ID: 90349399-701b-4ce9-affd-54978e664c51\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c4c027a02e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6xshnph7ptd9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/363f69bc-d1f2-4956-842d-980aba640091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/363f69bc-d1f2-4956-842d-980aba640091\",\n      \"name\": \"363f69bc-d1f2-4956-842d-980aba640091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:02:55.100233745Z\",\n      \"endTime\": \"2026-10-21T21:21:58.935771192Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1s0i5w0t2h9v3h-np-1-br8d2-5c77c: Failed: virtualmachine failed to create or update. err: failed to get existing resource fips-enabled-6xshnph7ptd9--managed/j1s0i5w0t2h9v3h-np-1-br8d2-5c77c (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-6xshnph7ptd9--managed/providers/Microsoft.Compute/virtualMachines/j1s0i5w0t2h9v3h-np-1-br8d2-5c77c\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https:... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/363f69bc-d1f2-4956-842d-980aba640091\n            correlation ID: 90349399-701b-4ce9-affd-54978e664c51\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16dd5f842340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-h4vdzmhsx8jj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/214af887-1dd9-4618-9147-3df5b406eba9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/214af887-1dd9-4618-9147-3df5b406eba9\",\n      \"name\": \"214af887-1dd9-4618-9147-3df5b406eba9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:02:58.414383168Z\",\n      \"endTime\": \"2026-10-21T21:21:59.161334523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-4x84k-cvs95: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-h4vdzmhsx8jj--managed/v20260901-np-1-4x84k-cvs95-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-h4vdzmhsx8jj--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-4x84k-cvs95-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/214af887-1dd9-4618-9147-3df5b406eba9\n            correlation ID: 6a1aa67d-c740-4581-b0a0-1116fc9156bb\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16dd5f842340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-h4vdzmhsx8jj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/214af887-1dd9-4618-9147-3df5b406eba9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/214af887-1dd9-4618-9147-3df5b406eba9\",\n      \"name\": \"214af887-1dd9-4618-9147-3df5b406eba9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:02:58.414383168Z\",\n      \"endTime\": \"2026-10-21T21:21:59.161334523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-4x84k-cvs95: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-h4vdzmhsx8jj--managed/v20260901-np-1-4x84k-cvs95-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-h4vdzmhsx8jj--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-4x84k-cvs95-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/214af887-1dd9-4618-9147-3df5b406eba9\n            correlation ID: 6a1aa67d-c740-4581-b0a0-1116fc9156bb\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113004424761708544","occurred_at":"2026-10-21T20:28:07Z","signature_id":"62e368863bbcc4d0e6d7e181761adb670d95c541b9b9f4f1c203c695bc640cdc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113004424761708544","occurred_at":"2026-10-21T20:28:07Z","signature_id":"eae99a5e02270845d83cd4033315772488c1f9bebef7d1f69cf52c0d511eebf4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0e9c16ade6d351c6ad76f00bf3537f55905505199679274bea1149b0e7a9d4d6","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-mksjxrdvl2vg\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg, error: context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x24f636554f60\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-mksjxrdvl2vg\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ext-auth-cidr\\\" in resourcegroup=\\\"e2e-cidr-connectivity-mksjxrdvl2vg\\\" for cluster=\\\"cidr-connectivity-test\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x24f635e3a480\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x24f636554f60\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-mksjxrdvl2vg\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ext-auth-cidr\\\" in resourcegroup=\\\"e2e-cidr-connectivity-mksjxrdvl2vg\\\" for cluster=\\\"cidr-connectivity-test\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x24f635e3a480\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-mksjxrdvl2vg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113004424761708544","occurred_at":"2026-10-21T20:28:07Z","signature_id":"84c8140ab58f2b25f6754e16c741350d2c529bfc248d3e915d3fa133d19aee62","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e80c6328b26789c96b80e99cd2ab080022b57ad25a3d94f7efee3036bc6074cc","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' se... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14b6bdca84e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-gxdr6w65gk4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\",\n      \"name\": \"f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:02:30.132303735Z\",\n      \"endTime\": \"2026-10-21T21:21:38.466559997Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-z68fd-hf6g2: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-gxdr6w65gk4m--managed/2tjdfde52u2f4iccs1dt8sgim461diee_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-gxdr6w65gk4m--managed/providers/Microsoft.Compute/availabilitySets/2tjdfde52u2f4iccs1dt8sgim461diee_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' se... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\n            correlation ID: 3e8f1379-1d07-482e-b286-68789ae9a346\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14b6bdca84e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-gxdr6w65gk4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\",\n      \"name\": \"f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:02:30.132303735Z\",\n      \"endTime\": \"2026-10-21T21:21:38.466559997Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-z68fd-hf6g2: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-gxdr6w65gk4m--managed/2tjdfde52u2f4iccs1dt8sgim461diee_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-gxdr6w65gk4m--managed/providers/Microsoft.Compute/availabilitySets/2tjdfde52u2f4iccs1dt8sgim461diee_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' se... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8d1a217-26ee-4eb3-92f9-7a398b20fbf3\n            correlation ID: 3e8f1379-1d07-482e-b286-68789ae9a346\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113004424761708544","occurred_at":"2026-10-21T20:28:07Z","signature_id":"ce707a4b8158707ffa5f88887b86aadfbc25a1c5d2800c9822d35025da98cd7e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"3cf5fe49d90e05a66f71645fcbe5e3da3910b60652f8e3a01cda5d6bc448c23b","failure_pattern":"VerifyImagePulled(acrpullxt7r5cdx2wbm.azurecr.io/ubi-minimal) timed out after 2m0.005s: failed to list pods in namespace acr-pull-test-day2: context deadline exceeded","search_query":"VerifyImagePulled(acrpullxt7r5cdx2wbm.azurecr.io/ubi-minimal) timed out after 2m0.005s: failed to list pods in namespace acr-pull-test-day2: context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:346]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullxt7r5cdx2wbm.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfec25889960\u003e: \n    VerifyImagePulled(acrpullxt7r5cdx2wbm.azurecr.io/ubi-minimal) timed out after 2m0.005s: failed to list pods in namespace acr-pull-test-day2: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(acrpullxt7r5cdx2wbm.azurecr.io/ubi-minimal) timed out after 2m0.005s: failed to list pods in namespace acr-pull-test-day2: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xfec25889880\u003e{\n            msg: \"failed to list pods in namespace acr-pull-test-day2: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:346]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullxt7r5cdx2wbm.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfec25889960\u003e: \n    VerifyImagePulled(acrpullxt7r5cdx2wbm.azurecr.io/ubi-minimal) timed out after 2m0.005s: failed to list pods in namespace acr-pull-test-day2: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(acrpullxt7r5cdx2wbm.azurecr.io/ubi-minimal) timed out after 2m0.005s: failed to list pods in namespace acr-pull-test-day2: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xfec25889880\u003e{\n            msg: \"failed to list pods in namespace acr-pull-test-day2: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113004424761708544","occurred_at":"2026-10-21T20:28:07Z","signature_id":"2e49401c40ba2a6fe1f21f13152cc8f3a36f62bc44172f331ddffb136e39e818","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}