{"meta":{"start_date":"2026-03-16","end_date":"2026-03-22","start_at":"2026-03-16T00:00:00Z","end_at":"2026-03-23T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T19:53:13Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":45,"failed_runs":24,"raw_occurrences":29,"matched_occurrences":29,"runs_affected":24},"rows":[{"environment":"int","failure_pattern_id":"8e73d7b981dbeda7884910f35c5fceffb07b4492f8382a34d5d73e5d1dcbd8f0","failure_pattern":"Value for field 'CommonName' failed to satisfy matcher.","search_query":"Value for field 'CommonName' failed to satisfy matcher.","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":15.555555555555555,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,3,4,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:98]: expect certificate to be issued by Microsoft\nValue for field 'CommonName' failed to satisfy matcher.\nExpected\n    \u003cstring\u003e: Microsoft TLS G2 RSA CA OCSP 02\nto match regular expression\n    \u003cstring\u003e: Microsoft\\sAzure\\sRSA\\sTLS\\sIssuing\\sCA\\s[0-9]+\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:98]: expect certificate to be issued by Microsoft\nValue for field 'CommonName' failed to satisfy matcher.\nExpected\n    \u003cstring\u003e: Microsoft TLS G2 RSA CA OCSP 02\nto match regular expression\n    \u003cstring\u003e: Microsoft\\sAzure\\sRSA\\sTLS\\sIssuing\\sCA\\s[0-9]+","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:98]: expect certificate to be issued by Microsoft\nValue for field 'CommonName' failed to satisfy matcher.\nExpected\n    \u003cstring\u003e: Microsoft TLS G2 RSA CA OCSP 04\nto match regular expression\n    \u003cstring\u003e: Microsoft\\sAzure\\sRSA\\sTLS\\sIssuing\\sCA\\s[0-9]+\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:98]: expect certificate to be issued by Microsoft\nValue for field 'CommonName' failed to satisfy matcher.\nExpected\n    \u003cstring\u003e: Microsoft TLS G2 RSA CA OCSP 04\nto match regular expression\n    \u003cstring\u003e: Microsoft\\sAzure\\sRSA\\sTLS\\sIssuing\\sCA\\s[0-9]+"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033800140292100096","occurred_at":"2026-03-17T06:58:34Z","signature_id":"88846fc125915ce568b3c08dba40e56fec6d1bfe049e7e40af9937676c052d6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033776026961580032","occurred_at":"2026-03-17T05:22:45Z","signature_id":"862ebd625a899ca912577648bc96afe8b5a068d268e06d7940da7e209089d2ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033745294096601088","occurred_at":"2026-03-17T03:20:37Z","signature_id":"862ebd625a899ca912577648bc96afe8b5a068d268e06d7940da7e209089d2ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033717274690457600","occurred_at":"2026-03-17T01:29:17Z","signature_id":"88846fc125915ce568b3c08dba40e56fec6d1bfe049e7e40af9937676c052d6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033669478067212288","occurred_at":"2026-03-16T22:19:21Z","signature_id":"88846fc125915ce568b3c08dba40e56fec6d1bfe049e7e40af9937676c052d6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033630670688882688","occurred_at":"2026-03-16T19:45:09Z","signature_id":"88846fc125915ce568b3c08dba40e56fec6d1bfe049e7e40af9937676c052d6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033613210036211712","occurred_at":"2026-03-16T18:35:46Z","signature_id":"862ebd625a899ca912577648bc96afe8b5a068d268e06d7940da7e209089d2ac","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"da4dfcdd7d527da38178deb1e7df2aa55ec7757189a3485be6bfb2ef4d4197ba","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while credentials are being revoked; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":11.11111111111111,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,1,1,0,0,1,2,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:249]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c14240\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while credentials are being revoked\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0016c4840\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Retry-After\": [\"10\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"7f7535b0-0fe4-48cd-881d-3b603716e11e\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/5788f641-fa93-4db7-9a91-aa50926905b1\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Date\": [\n                        \"Sat, 21 Mar 2026 18:38:33 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"347\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"41\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: E68D8F14163147F394B87CFEFB2E608F Ref B: CHI261140514060 Ref C: 2026-03-21T18:38:33Z\",\n                    ],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"18bf8c50-380a-4b05-815d-339e77cb569e\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260321T183833Z:18bf8c50-380a-4b05-815d-339e77cb569e\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000c14140\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 115, 32, 97, 114, 101, 32, 98, 101, 105, 110, 103, 32, 114, 101, 118, 111, 107, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 101, 98, 99, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:249]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c14240\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while credentials are being revoked\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-j274qw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-pnlzgg\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0016c4840\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Retry-After\": [\"10\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"7f7535b0-0fe4-48cd-881d-3b603716e11e\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/5788f641-fa93-4db7-9a91-aa50926905b1\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Date\": [\n                        \"Sat, 21 Mar 2026 18:38:33 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"347\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"41\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: E68D8F14163147F394B87CFEFB2E608F Ref B: CHI261140514060 Ref C: 2026-03-21T18:38:33Z\",\n                    ],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"18bf8c50-380a-4b05-815d-339e77cb569e\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260321T183833Z:18bf8c50-380a-4b05-815d-339e77cb569e\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000c14140\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 115, 32, 97, 114, 101, 32, 98, 101, 105, 110, 103, 32, 114, 101, 118, 111, 107, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 101, 98, 99, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:249]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00059fc00\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while credentials are being revoked\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000e10750\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"347\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/canadacentral/4dc1d6f1-e5cc-43f5-8c47-7eef96b2e0f5\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"cece0209-d710-4c31-bab2-21d80f15a352\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"CANADACENTRAL:20260321T050435Z:cece0209-d710-4c31-bab2-21d80f15a352\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Ms-Request-Id\": [\n                        \"cad83257-e003-422e-acaf-729a0dee5f00\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"42\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: F1D21E1D4E7448A9A043A7A6C7CD9F5B Ref B: CHI261140511062 Ref C: 2026-03-21T05:04:35Z\",\n                    ],\n                    \"Date\": [\n                        \"Sat, 21 Mar 2026 05:04:34 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00059fae0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 115, 32, 97, 114, 101, 32, 98, 101, 105, 110, 103, 32, 114, 101, 118, 111, 107, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:249]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00059fc00\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while credentials are being revoked\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2kc5ht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-79p5rk\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000e10750\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"347\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/canadacentral/4dc1d6f1-e5cc-43f5-8c47-7eef96b2e0f5\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"cece0209-d710-4c31-bab2-21d80f15a352\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"CANADACENTRAL:20260321T050435Z:cece0209-d710-4c31-bab2-21d80f15a352\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Ms-Request-Id\": [\n                        \"cad83257-e003-422e-acaf-729a0dee5f00\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"42\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: F1D21E1D4E7448A9A043A7A6C7CD9F5B Ref B: CHI261140511062 Ref C: 2026-03-21T05:04:35Z\",\n                    ],\n                    \"Date\": [\n                        \"Sat, 21 Mar 2026 05:04:34 GMT\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00059fae0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 115, 32, 97, 114, 101, 32, 98, 101, 105, 110, 103, 32, 114, 101, 118, 111, 107, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:249]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001328a20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while credentials are being revoked\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000e088d0\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"CANADACENTRAL:20260320T134725Z:e42c5286-5084-41f1-8a26-9a3abb7f5b29\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: AD1ED10A3FBE45A087E330C7FE2B7912 Ref B: CHI261140501029 Ref C: 2026-03-20T13:47:24Z\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 20 Mar 2026 13:47:25 GMT\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Request-Id\": [\n                        \"2d49c19c-df7f-4146-a1d8-66a45be53c4a\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"e42c5286-5084-41f1-8a26-9a3abb7f5b29\",\n                    ],\n                    \"Content-Length\": [\"347\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"60\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/canadacentral/698f44ec-1e20-4759-89d4-414a5c12bfb9\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0013286e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 115, 32, 97, 114, 101, 32, 98, 101, 105, 110, 103, 32, 114, 101, 118, 111, 107, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:249]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001328a20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while credentials are being revoked\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-v9l57f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-hmbfpm\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000e088d0\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"CANADACENTRAL:20260320T134725Z:e42c5286-5084-41f1-8a26-9a3abb7f5b29\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: AD1ED10A3FBE45A087E330C7FE2B7912 Ref B: CHI261140501029 Ref C: 2026-03-20T13:47:24Z\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 20 Mar 2026 13:47:25 GMT\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Request-Id\": [\n                        \"2d49c19c-df7f-4146-a1d8-66a45be53c4a\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"e42c5286-5084-41f1-8a26-9a3abb7f5b29\",\n                    ],\n                    \"Content-Length\": [\"347\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"60\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/canadacentral/698f44ec-1e20-4759-89d4-414a5c12bfb9\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0013286e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 115, 32, 97, 114, 101, 32, 98, 101, 105, 110, 103, 32, 114, 101, 118, 111, 107, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035414651126157312","occurred_at":"2026-03-21T17:54:03Z","signature_id":"ce795bbd665a65f6059723382a04a2a9b1fcd49c173c2c1b633d12c050531191","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035211817340375040","occurred_at":"2026-03-21T04:28:02Z","signature_id":"00e94ae74830760a52f61ec2e9dbc00f9451e6426464fe91ea118c6556abe3fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2034980877347328000","occurred_at":"2026-03-20T13:10:23Z","signature_id":"870aa18f94cc6be5552b90cc07bb57d5d042cf536d60292ecc206b189a3674ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033974052363702272","occurred_at":"2026-03-17T18:29:38Z","signature_id":"a0376641748e0bdfaca4ad7d35ff3d5b28b20e02c39138e07d1966045d891277","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033669478067212288","occurred_at":"2026-03-16T22:19:21Z","signature_id":"3b8f6ad5effe6066a2932257ab7cdfec0ed5126cfe59272a1968f240b2523e15","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"67e7ef5fc3321b30883e9c59e138623f00ae8b9c469840450526d3ef9f9a38b2","failure_pattern":"not to be empty","search_query":"not to be empty","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":11.11111111111111,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,2,3,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:126]: Expected\n    \u003c[]*armredhatopenshifthcp.ImageDigestMirror | len:0, cap:0\u003e: nil\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:126]: Expected\n    \u003c[]*armredhatopenshifthcp.ImageDigestMirror | len:0, cap:0\u003e: nil\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035233237504102400","occurred_at":"2026-03-21T05:53:11Z","signature_id":"c2437d1f3013b7e5529f2ad9cba0f07c235e209120a5623fcb08599717d53a8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035211817340375040","occurred_at":"2026-03-21T04:28:02Z","signature_id":"c2437d1f3013b7e5529f2ad9cba0f07c235e209120a5623fcb08599717d53a8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035165939305025536","occurred_at":"2026-03-21T01:25:46Z","signature_id":"c2437d1f3013b7e5529f2ad9cba0f07c235e209120a5623fcb08599717d53a8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035139151292010496","occurred_at":"2026-03-20T23:39:19Z","signature_id":"c2437d1f3013b7e5529f2ad9cba0f07c235e209120a5623fcb08599717d53a8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035103366039736320","occurred_at":"2026-03-20T21:17:07Z","signature_id":"c2437d1f3013b7e5529f2ad9cba0f07c235e209120a5623fcb08599717d53a8d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7, error: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.444444444444445,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,1,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-03T21:16:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010129f0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-lfq2nh, error: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-NC4asT4v3-lfq2nh\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-lfq2nh, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-lfq2nh\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0005ae7b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-lfq2nh\",\n            },\n            \u003c*fmt.wrapError | 0xc0005b0500\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-lfq2nh\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010129f0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-lfq2nh, error: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-NC4asT4v3-lfq2nh\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-lfq2nh, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-lfq2nh\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0005ae7b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-lfq2nh\",\n            },\n            \u003c*fmt.wrapError | 0xc0005b0500\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-lfq2nh\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a168d0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7, error: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00077f3b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7\",\n            },\n            \u003c*fmt.wrapError | 0xc00047ad00\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a168d0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7, error: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00077f3b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-gpu-nodepool-NC4asT4v3-d9mvv7\",\n            },\n            \u003c*fmt.wrapError | 0xc00047ad00\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cluster-gpu-np\\\" in resourcegroup=\\\"rg-gpu-nodepool-NC4asT4v3-d9mvv7\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2034275094439661568","occurred_at":"2026-03-18T14:25:52Z","signature_id":"cf01bdc32e6756ba795854c3c23a769544fc29ba4b0bd7c28659eabeb232bf36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033502721323044864","occurred_at":"2026-03-16T11:16:40Z","signature_id":"70ec36ea594510ffed5711c51089f1145efc3e07e08ca047ffd887d6affc116f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8d06cad82b6f76b897e08d4e2537f6c6d4cc616ff64635c4d37783734841ca74","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hgqhmg\" in resourcegroup=\"admin-credential-lifecycle-test-hsckzg\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.444444444444445,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-15T21:48:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ab30b0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-x7n2mn\" in resourcegroup=\"admin-credential-lifecycle-test-j72vnp\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-x7n2mn in resource group admin-credential-lifecycle-test-j72vnp, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-x7n2mn\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j72vnp\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-x7n2mn in resource group admin-credential-lifecycle-test-j72vnp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b08d70\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-x7n2mn in resource group admin-credential-lifecycle-test-j72vnp\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ab30b0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-x7n2mn\" in resourcegroup=\"admin-credential-lifecycle-test-j72vnp\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-x7n2mn in resource group admin-credential-lifecycle-test-j72vnp, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-x7n2mn\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j72vnp\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-x7n2mn in resource group admin-credential-lifecycle-test-j72vnp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b08d70\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-x7n2mn in resource group admin-credential-lifecycle-test-j72vnp\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0004d1ef0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-hgqhmg\" in resourcegroup=\"admin-credential-lifecycle-test-hsckzg\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-hgqhmg\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-hsckzg\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008a2580\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0004d1ef0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-hgqhmg\" in resourcegroup=\"admin-credential-lifecycle-test-hsckzg\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-hgqhmg\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-hsckzg\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008a2580\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-hgqhmg in resource group admin-credential-lifecycle-test-hsckzg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033398635554672640","occurred_at":"2026-03-16T04:23:02Z","signature_id":"317b0c88856783e53f910c2b9d9c34d9b2eca909a33255801df14a74f8280ece","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033353364464472064","occurred_at":"2026-03-16T01:23:14Z","signature_id":"ecee7333d080d415b5805335b91ce6de1457c5dd705c021d60c7c100cb6ca00b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e457d9ded8188e2e5d3bf94632039456ab0d3f40aab17c66a163827314f30476","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.444444444444445,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[1,2,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-15T05:21:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 2 should fail after revocation, last error: client rate limiter Wait returned an error: context deadline exceeded\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 2 should fail after revocation, last error: client rate limiter Wait returned an error: context deadline exceeded\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: client rate limiter Wait returned an error: context deadline exceeded\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: client rate limiter Wait returned an error: context deadline exceeded\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033502721323044864","occurred_at":"2026-03-16T11:16:40Z","signature_id":"deb11aa792f1cd28be7abc11a6773e5db8439cf03fc59f97afa43ccc3f2f133b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033422880192073728","occurred_at":"2026-03-16T05:59:28Z","signature_id":"59c642fdc88f1fb11d46b08cbd20bafcf3e7f95194d6e3ce38cdbe2bc337ceb7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.444444444444445,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,1,0,0,1,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-06T08:15:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0010d4620\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-bvxmn.apps.aro.f0b8a1v8r9h2u2b.os6f.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-bvxmn.apps.aro.f0b8a1v8r9h2u2b.os6f.uksouth.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0xc0004eb7d0\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-bvxmn.apps.aro.f0b8a1v8r9h2u2b.os6f.uksouth.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0xc0004eb7a0\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 7, 24, 73, 129, 6, 189, 3, 123, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 48, 51, 50, 49, 48, 52, 51, 52, 52, 53, 90, 23, 13, 50, 55, 48, 51, 50, 49, 48, 52, 51, 52, 52, 53, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 146, 154, 117, 19, 224, 169, 186, 58, 230, 251, 170, 128, 12, 92, 140, 43, 150, 232, 193, 85, 125, 186, 192, 131, 156, 99, 64, 239, 192, 84, 127, 85, 189, 114, 253, 212, 191, 60, 158, 147, 15, 141, 170, 204, 150, 241, 177, 59, 102, 231, 52, 19, 133, 224, 93, 111, 167, 251, 18, 180, 5, 60, 20, 189, 127, 178, 135, 78, 203, 93, 51, 114, 109, 53, 192, 55, 251, 146, 34, 89, 80, 242, 191, 212, 11, 251, 92, 82, 168, 202, 204, 236, 238, 218, 103, 249, 2, 3, 255, 188, 85, 122, 226, 237, 154, 87, 201, 92, 30, 248, 58, 219, 162, 210, 53, 157, 162, 250, 13, 20, 212, 154, 176, 116, 27, 254, 245, 181, 150, 10, 59, 176, 29, 71, 28, 57, 182, 49, 5, 158, 179, 170, 172, 231, 206, 41, 33, 116, 210, 92, 202, 59, 106, 58, 27, 239, 134, 65, 182, 26, 80, 0, 100, 43, 209, 139, 239, 124, 205, 146, 80, 246, 172, 110, 201, 17, 254, 203, 109, 163, 106, 211, 173, 4, 189, 234, 251, 98, 73, 158, 174, 247, 53, 180, 27, 93, 250, 50, 117, 20, 151, 173, 185, 134, 194, 240, 222, 51, 170, 47, 27, 54, 203, 225, 106, 136, 18, 33, 228, 201, 33, 155, 37, 143, 87, 148, 82, 254, 251, 98, 218, 65, 242, 232, 212, 33, 201, 225, 32, 190, 109, 194, 130, 84, 27, 198, 103, 231, 91, 141, 11, 103, 87, 81, 176, 213, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 249, 194, 155, 225, 108, 42, 222, 162, 76, 217, 84, 226, 207, 104, 26, 175, 56, 56, 202, 130, 109, 22, 8, 88, 98, 181, 203, 29, 149, 64, 217, 87, 135, 80, 81, 243, 101, 197, 164, 115, 41, 191, 99, 167, 60, 215, 132, 238, 202, 168, 104, 246, 147, 193, 228, 19, 132, 36, 200, 162, 230, 233, 91, 223, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 152, 169, 77, 135, 116, 37, 109, 16, 231, 51, 94, 199, 44, 186, 126, 209, 145, 141, 143, 17, 91, 108, 156, 142, 31, 143, 94, 218, 152, 163, 241, 6, 207, 199, 132, 180, 158, 186, 168, 185, 207, 94, 136, 94, 0, 61, 160, 112, 246, 162, 1, 251, 107, 226, 144, 122, 247, 227, 95, 232, 130, 0, 122, 143, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 102, 48, 98, 56, 97, 49, 118, 56, 114, 57, 104, 50, 117, 50, 98, 46, 111, 115, 54, 102, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 109, 149, 191, 123, 150, 135, 112, 137, 56, 3, 119, 61, 94, 253, 20, 213, 31, 8, 90, 58,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0010d4620\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-bvxmn.apps.aro.f0b8a1v8r9h2u2b.os6f.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-bvxmn.apps.aro.f0b8a1v8r9h2u2b.os6f.uksouth.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0xc0004eb7d0\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-bvxmn.apps.aro.f0b8a1v8r9h2u2b.os6f.uksouth.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0xc0004eb7a0\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 7, 24, 73, 129, 6, 189, 3, 123, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 48, 51, 50, 49, 48, 52, 51, 52, 52, 53, 90, 23, 13, 50, 55, 48, 51, 50, 49, 48, 52, 51, 52, 52, 53, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 146, 154, 117, 19, 224, 169, 186, 58, 230, 251, 170, 128, 12, 92, 140, 43, 150, 232, 193, 85, 125, 186, 192, 131, 156, 99, 64, 239, 192, 84, 127, 85, 189, 114, 253, 212, 191, 60, 158, 147, 15, 141, 170, 204, 150, 241, 177, 59, 102, 231, 52, 19, 133, 224, 93, 111, 167, 251, 18, 180, 5, 60, 20, 189, 127, 178, 135, 78, 203, 93, 51, 114, 109, 53, 192, 55, 251, 146, 34, 89, 80, 242, 191, 212, 11, 251, 92, 82, 168, 202, 204, 236, 238, 218, 103, 249, 2, 3, 255, 188, 85, 122, 226, 237, 154, 87, 201, 92, 30, 248, 58, 219, 162, 210, 53, 157, 162, 250, 13, 20, 212, 154, 176, 116, 27, 254, 245, 181, 150, 10, 59, 176, 29, 71, 28, 57, 182, 49, 5, 158, 179, 170, 172, 231, 206, 41, 33, 116, 210, 92, 202, 59, 106, 58, 27, 239, 134, 65, 182, 26, 80, 0, 100, 43, 209, 139, 239, 124, 205, 146, 80, 246, 172, 110, 201, 17, 254, 203, 109, 163, 106, 211, 173, 4, 189, 234, 251, 98, 73, 158, 174, 247, 53, 180, 27, 93, 250, 50, 117, 20, 151, 173, 185, 134, 194, 240, 222, 51, 170, 47, 27, 54, 203, 225, 106, 136, 18, 33, 228, 201, 33, 155, 37, 143, 87, 148, 82, 254, 251, 98, 218, 65, 242, 232, 212, 33, 201, 225, 32, 190, 109, 194, 130, 84, 27, 198, 103, 231, 91, 141, 11, 103, 87, 81, 176, 213, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 249, 194, 155, 225, 108, 42, 222, 162, 76, 217, 84, 226, 207, 104, 26, 175, 56, 56, 202, 130, 109, 22, 8, 88, 98, 181, 203, 29, 149, 64, 217, 87, 135, 80, 81, 243, 101, 197, 164, 115, 41, 191, 99, 167, 60, 215, 132, 238, 202, 168, 104, 246, 147, 193, 228, 19, 132, 36, 200, 162, 230, 233, 91, 223, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 152, 169, 77, 135, 116, 37, 109, 16, 231, 51, 94, 199, 44, 186, 126, 209, 145, 141, 143, 17, 91, 108, 156, 142, 31, 143, 94, 218, 152, 163, 241, 6, 207, 199, 132, 180, 158, 186, 168, 185, 207, 94, 136, 94, 0, 61, 160, 112, 246, 162, 1, 251, 107, 226, 144, 122, 247, 227, 95, 232, 130, 0, 122, 143, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 102, 48, 98, 56, 97, 49, 118, 56, 114, 57, 104, 50, 117, 50, 98, 46, 111, 115, 54, 102, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 109, 149, 191, 123, 150, 135, 112, 137, 56, 3, 119, 61, 94, 253, 20, 213, 31, 8, 90, 58,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e12c00\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-vcfw6.apps.aro.q9c7h6e2w7p6c4m.f8k5.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-vcfw6.apps.aro.q9c7h6e2w7p6c4m.f8k5.uksouth.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0xc00092cc30\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-vcfw6.apps.aro.q9c7h6e2w7p6c4m.f8k5.uksouth.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0xc00092cc00\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 53, 224, 152, 96, 21, 167, 217, 201, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 48, 51, 49, 56, 50, 51, 50, 55, 53, 57, 90, 23, 13, 50, 55, 48, 51, 49, 56, 50, 51, 50, 55, 53, 57, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 137, 78, 205, 85, 199, 248, 179, 129, 134, 68, 36, 203, 162, 10, 128, 215, 1, 74, 91, 116, 62, 126, 190, 241, 168, 144, 110, 35, 152, 39, 215, 76, 22, 233, 158, 0, 42, 191, 24, 163, 76, 48, 4, 241, 70, 75, 61, 149, 161, 248, 157, 216, 21, 97, 6, 208, 183, 156, 81, 34, 80, 227, 244, 251, 169, 161, 34, 95, 149, 126, 166, 106, 95, 207, 0, 55, 48, 132, 207, 93, 64, 160, 113, 174, 79, 36, 191, 107, 48, 149, 188, 225, 46, 202, 49, 6, 115, 139, 174, 196, 98, 132, 244, 3, 95, 111, 27, 160, 244, 243, 101, 147, 126, 244, 41, 189, 115, 188, 237, 70, 244, 127, 34, 28, 10, 173, 143, 205, 149, 180, 184, 254, 16, 32, 197, 203, 14, 228, 97, 163, 198, 132, 241, 221, 157, 34, 154, 235, 73, 134, 229, 105, 125, 174, 102, 235, 223, 197, 12, 100, 163, 230, 125, 96, 197, 168, 218, 160, 196, 159, 201, 202, 210, 79, 77, 134, 201, 49, 105, 250, 24, 219, 64, 150, 144, 124, 154, 243, 147, 75, 141, 100, 225, 206, 150, 62, 86, 45, 93, 177, 56, 98, 23, 180, 61, 49, 84, 50, 239, 235, 175, 184, 173, 152, 119, 60, 156, 147, 81, 202, 133, 229, 88, 224, 134, 204, 105, 85, 128, 141, 211, 162, 111, 199, 22, 6, 13, 11, 194, 247, 30, 224, 113, 60, 122, 51, 213, 165, 253, 6, 36, 220, 151, 54, 143, 101, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 220, 49, 223, 226, 128, 113, 91, 117, 229, 53, 163, 156, 247, 135, 53, 234, 93, 79, 175, 247, 41, 93, 249, 185, 64, 107, 7, 233, 180, 254, 202, 192, 225, 52, 17, 190, 177, 193, 99, 101, 55, 231, 6, 26, 224, 3, 87, 191, 238, 145, 90, 12, 79, 227, 221, 18, 52, 131, 176, 103, 254, 114, 200, 86, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 96, 77, 79, 254, 38, 90, 139, 10, 37, 247, 206, 194, 82, 102, 238, 63, 128, 238, 166, 22, 251, 172, 206, 77, 204, 161, 50, 124, 66, 183, 196, 237, 163, 37, 121, 3, 202, 211, 226, 127, 10, 171, 226, 202, 104, 170, 99, 153, 62, 67, 212, 234, 206, 141, 144, 4, 35, 68, 228, 3, 69, 187, 249, 191, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 113, 57, 99, 55, 104, 54, 101, 50, 119, 55, 112, 54, 99, 52, 109, 46, 102, 56, 107, 53, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 41, 198, 145, 169, 222, 82, 247, 79, 220, 178, 90, 26, 252, 136, 89, 54, 145, 184, 38, 64, 219,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e12c00\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-vcfw6.apps.aro.q9c7h6e2w7p6c4m.f8k5.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-vcfw6.apps.aro.q9c7h6e2w7p6c4m.f8k5.uksouth.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0xc00092cc30\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-vcfw6.apps.aro.q9c7h6e2w7p6c4m.f8k5.uksouth.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0xc00092cc00\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 53, 224, 152, 96, 21, 167, 217, 201, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 48, 51, 49, 56, 50, 51, 50, 55, 53, 57, 90, 23, 13, 50, 55, 48, 51, 49, 56, 50, 51, 50, 55, 53, 57, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 137, 78, 205, 85, 199, 248, 179, 129, 134, 68, 36, 203, 162, 10, 128, 215, 1, 74, 91, 116, 62, 126, 190, 241, 168, 144, 110, 35, 152, 39, 215, 76, 22, 233, 158, 0, 42, 191, 24, 163, 76, 48, 4, 241, 70, 75, 61, 149, 161, 248, 157, 216, 21, 97, 6, 208, 183, 156, 81, 34, 80, 227, 244, 251, 169, 161, 34, 95, 149, 126, 166, 106, 95, 207, 0, 55, 48, 132, 207, 93, 64, 160, 113, 174, 79, 36, 191, 107, 48, 149, 188, 225, 46, 202, 49, 6, 115, 139, 174, 196, 98, 132, 244, 3, 95, 111, 27, 160, 244, 243, 101, 147, 126, 244, 41, 189, 115, 188, 237, 70, 244, 127, 34, 28, 10, 173, 143, 205, 149, 180, 184, 254, 16, 32, 197, 203, 14, 228, 97, 163, 198, 132, 241, 221, 157, 34, 154, 235, 73, 134, 229, 105, 125, 174, 102, 235, 223, 197, 12, 100, 163, 230, 125, 96, 197, 168, 218, 160, 196, 159, 201, 202, 210, 79, 77, 134, 201, 49, 105, 250, 24, 219, 64, 150, 144, 124, 154, 243, 147, 75, 141, 100, 225, 206, 150, 62, 86, 45, 93, 177, 56, 98, 23, 180, 61, 49, 84, 50, 239, 235, 175, 184, 173, 152, 119, 60, 156, 147, 81, 202, 133, 229, 88, 224, 134, 204, 105, 85, 128, 141, 211, 162, 111, 199, 22, 6, 13, 11, 194, 247, 30, 224, 113, 60, 122, 51, 213, 165, 253, 6, 36, 220, 151, 54, 143, 101, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 220, 49, 223, 226, 128, 113, 91, 117, 229, 53, 163, 156, 247, 135, 53, 234, 93, 79, 175, 247, 41, 93, 249, 185, 64, 107, 7, 233, 180, 254, 202, 192, 225, 52, 17, 190, 177, 193, 99, 101, 55, 231, 6, 26, 224, 3, 87, 191, 238, 145, 90, 12, 79, 227, 221, 18, 52, 131, 176, 103, 254, 114, 200, 86, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 96, 77, 79, 254, 38, 90, 139, 10, 37, 247, 206, 194, 82, 102, 238, 63, 128, 238, 166, 22, 251, 172, 206, 77, 204, 161, 50, 124, 66, 183, 196, 237, 163, 37, 121, 3, 202, 211, 226, 127, 10, 171, 226, 202, 104, 170, 99, 153, 62, 67, 212, 234, 206, 141, 144, 4, 35, 68, 228, 3, 69, 187, 249, 191, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 113, 57, 99, 55, 104, 54, 101, 50, 119, 55, 112, 54, 99, 52, 109, 46, 102, 56, 107, 53, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 41, 198, 145, 169, 222, 82, 247, 79, 220, 178, 90, 26, 252, 136, 89, 54, 145, 184, 38, 64, 219,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035211817340375040","occurred_at":"2026-03-21T04:28:02Z","signature_id":"5226fcfcf98a541111562bac69d4fcfe6e233e120ebd94528d852070c4dc1fb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2034409619874385920","occurred_at":"2026-03-18T23:20:25Z","signature_id":"1c297e7958c546f4cd01ff04b0d907b692f300c618a422531f761d1849e3d5d5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d0993ee7c174e66d21d6873cdcbb83cae1352c1482345816cea0f68947670c76","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":18,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-08T23:23:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["r.go:284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f55ac3c5a60, 0xc000636bd0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8278de0?, 0xc000636bd0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 97\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 09:18:44.139\n  \"ts\"=\"2026-03-18 09:18:44.139918\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 09:18:44.139996\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-52ql4t\" \"resourceGroup\"=\"customer-rg-rfp52c\"\n  \"ts\"=\"2026-03-18 09:18:44.346235\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-52ql4t\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 09:18:45.945878\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-jwd5jl\" \"resourceGroup\"=\"customer-rg-rfp52c\"\n  \"ts\"=\"2026-03-18 09:18:46.101260\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-jwd5jl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 09:18:47.985220\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 09:18:47.985336\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 09:18:47.985396\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 09:18:47.986501\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\"\n  \"ts\"=\"2026-03-18 09:18:47.986522\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-60\"\n  \"ts\"=\"2026-03-18 09:18:47.986539\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-117\"\n  \"ts\"=\"2026-03-18 09:18:47.986585\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-rfp52c\"\n  \"ts\"=\"2026-03-18 09:18:48.234851\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 09:18:48.317393\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 09:18:48.362861\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-60\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 09:18:48.365533\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-117\" \"deployment\"=\"msi-deployment-117\"\n  \"ts\"=\"2026-03-18 09:18:48.465594\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-jwd5jl\"\n  \"ts\"=\"2026-03-18 09:18:48.501722\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-60\" \"deployment\"=\"msi-deployment-60\"\n  \"ts\"=\"2026-03-18 09:18:48.625330\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 09:18:48.660006\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 09:18:48.790139\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"customer-infra-basic-hcp-cluster-52ql4t\"\n  \"ts\"=\"2026-03-18 09:18:48.876784\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"msi-deployment-10\"\n  \"ts\"=\"2026-03-18 09:18:49.020248\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 09:18:49.020387\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 09:18:49.020486\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-rfp52c\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nr.go:284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f55ac3c5a60, 0xc000636bd0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8278de0?, 0xc000636bd0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 97\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 09:18:44.139\n  \"ts\"=\"2026-03-18 09:18:44.139918\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 09:18:44.139996\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-52ql4t\" \"resourceGroup\"=\"customer-rg-rfp52c\"\n  \"ts\"=\"2026-03-18 09:18:44.346235\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-52ql4t\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 09:18:45.945878\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-jwd5jl\" \"resourceGroup\"=\"customer-rg-rfp52c\"\n  \"ts\"=\"2026-03-18 09:18:46.101260\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-jwd5jl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 09:18:47.985220\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 09:18:47.985336\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 09:18:47.985396\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 09:18:47.986501\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\"\n  \"ts\"=\"2026-03-18 09:18:47.986522\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-60\"\n  \"ts\"=\"2026-03-18 09:18:47.986539\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-117\"\n  \"ts\"=\"2026-03-18 09:18:47.986585\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-rfp52c\"\n  \"ts\"=\"2026-03-18 09:18:48.234851\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 09:18:48.317393\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 09:18:48.362861\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-60\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 09:18:48.365533\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-117\" \"deployment\"=\"msi-deployment-117\"\n  \"ts\"=\"2026-03-18 09:18:48.465594\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-jwd5jl\"\n  \"ts\"=\"2026-03-18 09:18:48.501722\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-60\" \"deployment\"=\"msi-deployment-60\"\n  \"ts\"=\"2026-03-18 09:18:48.625330\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 09:18:48.660006\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 09:18:48.790139\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rfp52c\" \"deployment\"=\"customer-infra-basic-hcp-cluster-52ql4t\"\n  \"ts\"=\"2026-03-18 09:18:48.876784\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"msi-deployment-10\"\n  \"ts\"=\"2026-03-18 09:18:49.020248\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 09:18:49.020387\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 09:18:49.020486\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-rfp52c\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2034172422873157632","occurred_at":"2026-03-18T07:37:53Z","signature_id":"0d63ba22e2f593ff6ae45fd281fa7135f6292cb75ddea6edd61126f972d7c186","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"3d6e06fd3d1b39873e90a0812ecb95c80f96fa8ca33ea058b920192258ebe5df","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-02-23T18:23:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:254]: New admin credentials should work after revocation\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000d20db0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d37080\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0xc000d37060\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0xc000c62be0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xc000d83500\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0xc000d83360\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0xc0009d0280\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:254]: New admin credentials should work after revocation\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000d20db0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d37080\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0xc000d37060\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0xc000c62be0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xc000d83500\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0xc000d83360\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0xc0009d0280\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2034017373396144128","occurred_at":"2026-03-17T21:21:46Z","signature_id":"6edaccb76e7583910e2fca2c774b48cd30f74e930c511c28e4b6efcdedc7333e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"c895003ef295c772de4b3f116f645dc16aebdfc43ff0418796c18bf9575c19bc","failure_pattern":"found 1 managed resource groups left behind HCP clusters in cluster-listing-2s9rvc","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111f080\u003e: \n    found 1 managed resource groups left behind HCP clusters in cluster-listing-2s9rvc\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc00135d4d0\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-2s9rvc\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111f080\u003e: \n    found 1 managed resource groups left behind HCP clusters in cluster-listing-2s9rvc\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc00135d4d0\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-2s9rvc\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033776026961580032","occurred_at":"2026-03-17T05:22:45Z","signature_id":"fdf3670ac3c60dbfcb7873fcfa75fb94a3251528467031ad51c7c8f7dfd3ccd0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: failed waiting for cluster=\"cluster-zstream-4-19-pr8tbf\" in resourcegroup=\"rg-zstream-upgrade-4-19-8h6nq4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":3,"prior_window_starts":["2026-02-23","2026-03-02","2026-03-09"],"prior_runs_affected":17,"prior_last_seen_at":"2026-03-14T00:12:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00052c2d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-pr8tbf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: failed waiting for cluster=\"cluster-zstream-4-19-pr8tbf\" in resourcegroup=\"rg-zstream-upgrade-4-19-8h6nq4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-pr8tbf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: failed waiting for cluster=\\\"cluster-zstream-4-19-pr8tbf\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-8h6nq4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010a1340\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00052c270\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-pr8tbf\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-8h6nq4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010a1340\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00052c2d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-pr8tbf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: failed waiting for cluster=\"cluster-zstream-4-19-pr8tbf\" in resourcegroup=\"rg-zstream-upgrade-4-19-8h6nq4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-pr8tbf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: failed waiting for cluster=\\\"cluster-zstream-4-19-pr8tbf\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-8h6nq4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010a1340\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00052c270\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-pr8tbf\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-8h6nq4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010a1340\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pr8tbf in resource group rg-zstream-upgrade-4-19-8h6nq4\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2035569887107092480","occurred_at":"2026-03-22T04:10:54Z","signature_id":"42cf4815b50f6319bbcc5dacecce2df45af5393d7a49e76db6cf5b3687386be4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}