{"meta":{"start_date":"2026-04-04","end_date":"2026-04-04","start_at":"2026-04-04T00:00:00Z","end_at":"2026-04-05T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T13:40:39Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":1,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":1},"rows":[{"environment":"stg","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-19-ds9qjd\" in resourcegroup=\"rg-zstream-upgrade-4-19-hcc6mp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-ds9qjd in resource group rg-zstream-upgrade-4-19-hcc6mp, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-03-29..2026-04-04 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-23T02:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cf6060\u003e: \n    failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kdhhj9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-kdhhj9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-kdhhj9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-kdhhj9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e5e860\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-kdhhj9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cf6060\u003e: \n    failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kdhhj9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-kdhhj9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-kdhhj9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-kdhhj9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e5e860\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-kdhhj9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001622fc0\u003e: \n    failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tcch44\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-tcch44, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-tcch44\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-tcch44, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000886b10\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-tcch44\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001622fc0\u003e: \n    failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tcch44\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-tcch44, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-tcch44\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-tcch44, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000886b10\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-tcch44\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009c8060\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-rn4cgn\" in resourcegroup=\"rg-np-version-upgrade-rn4cgn-2nnmgb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-rn4cgn in resource group rg-np-version-upgrade-rn4cgn-2nnmgb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"np-version-upgrade-cluster-rn4cgn\\\" in resourcegroup=\\\"rg-np-version-upgrade-rn4cgn-2nnmgb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-rn4cgn in resource group rg-np-version-upgrade-rn4cgn-2nnmgb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015208f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-rn4cgn in resource group rg-np-version-upgrade-rn4cgn-2nnmgb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009c8060\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-rn4cgn\" in resourcegroup=\"rg-np-version-upgrade-rn4cgn-2nnmgb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-rn4cgn in resource group rg-np-version-upgrade-rn4cgn-2nnmgb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"np-version-upgrade-cluster-rn4cgn\\\" in resourcegroup=\\\"rg-np-version-upgrade-rn4cgn-2nnmgb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-rn4cgn in resource group rg-np-version-upgrade-rn4cgn-2nnmgb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015208f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-rn4cgn in resource group rg-np-version-upgrade-rn4cgn-2nnmgb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"33300053dbad9e28a7270a95a3547fb91c65af7d71efea9d3908c366d7c9e2f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"425163d3b07eeea206817fea9114ce231c7affa81d68cb7bdd6b0d630517c767","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"5a24aa7b5b52f72c55b7b10585864266c0ba8924e164a4fdd00dbfb199b8c224","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"ab9c2f45c3c3c691585d16285aa143f923279bb69be1ade0d1f5a95650ca968c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"c6f683aea5f50d72108b2d70a37325bdad079fb0b15160489207e7d2a5764760","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"d2c18c1ce7acab0b289b6521ca0f196b8a3fe81360e8f107f1880ea6a6437f48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"ebf05020222bb7411531b4209113a1628b22b3b8eda5ed2a04777e3327620d73","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[0,0,0,0,6,3,1],"trend_range":"2026-03-29..2026-04-04 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ef0e0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001357950\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"166\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 2EC63483A147487995A6328397F09705 Ref B: CHI261140505054 Ref C: 2026-04-04T02:44:04Z\",\n                    ],\n                    \"Date\": [\n                        \"Sat, 04 Apr 2026 02:44:04 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260404T024405Z:a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/52040d45-044c-4511-88b7-9733c92944b6\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Request-Id\": [\n                        \"12919637-123e-4c90-833e-437777c2019f\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0005eefa0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ef0e0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001357950\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"166\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 2EC63483A147487995A6328397F09705 Ref B: CHI261140505054 Ref C: 2026-04-04T02:44:04Z\",\n                    ],\n                    \"Date\": [\n                        \"Sat, 04 Apr 2026 02:44:04 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260404T024405Z:a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/52040d45-044c-4511-88b7-9733c92944b6\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Request-Id\": [\n                        \"12919637-123e-4c90-833e-437777c2019f\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0005eefa0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"d7688ab63a389ea09f8ad41fad1bbee654d86cd103c78d4ba4346ea44fd4f632","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"1663595c725a539d4430499c75496841a6e725e86c061ab925a6870046e0606e","failure_pattern":"UpdateHCPCluster20251223 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j9bcfb\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-29..2026-04-04 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:231]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ee840\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j9bcfb\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-j9bcfb\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008ba000\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:231]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ee840\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j9bcfb\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-j9bcfb\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008ba000\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"79be309de5c8ad4c6e57a1f38af62709c7522c3d70c281827d02832ac568702a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmmjkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-29..2026-04-04 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":15,"prior_last_seen_at":"2026-03-26T19:56:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013345a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmmjkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010337e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001334570\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010337e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013345a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmmjkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010337e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001334570\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010337e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"c9bea9c76d2740349bda369a455dda33830aa97646af9e628ce09a94bc8774fa","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}