{"meta":{"date":"2026-07-23","timezone":"UTC","generated_at":"2026-08-26T18:45:00Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":10,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2080427936950784000","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T23:00:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d5fd23a60893676016056c31ddb85249ff76013ce9631991b1dc0e3392f271ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2080427936950784000","occurred_at":"2026-07-23T23:00:47Z","failed_at":"e2e","signature_id":"703d20a7268c58a9e79f73180f6f920ace778269f7cc2f8e56f674ec9aaace7f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001e0aa0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9n27h62csfxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"name\": \"032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T23:04:31.918481763Z\",\n      \"endTime\": \"2026-07-23T23:23:43.694668207Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001e0aa0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9n27h62csfxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"name\": \"032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T23:04:31.918481763Z\",\n      \"endTime\": \"2026-07-23T23:23:43.694668207Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080316813475844096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-23T15:39:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080316368640544768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-23T15:37:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080267996604928000","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T12:25:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ac41e3e6cb9cbb38e341492c578b9bd2c819341d16355cefbdc518f60a16c519","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080267996604928000","occurred_at":"2026-07-23T12:25:14Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080267677162541056","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-23T12:23:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080188520843972608","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T07:09:26Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6e47b0333613324321d945844af1686ad8312e19d92fbce4cf09e4b5d742a056","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080188520843972608","occurred_at":"2026-07-23T07:09:26Z","failed_at":"e2e","signature_id":"463ea67d556eeb787e73f24b0e88b683d070445d3805c355c27706ee59576f5a","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00283c700\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5676c2nc84jf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"name\": \"81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T07:46:53.376501029Z\",\n      \"endTime\": \"2026-07-23T08:05:58.659570015Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00283c700\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5676c2nc84jf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"name\": \"81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T07:46:53.376501029Z\",\n      \"endTime\": \"2026-07-23T08:05:58.659570015Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"afd8cf0f67c25933f9c91107ef933e45f88c870c0ac629a314965b7817b9a600","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080188520843972608","occurred_at":"2026-07-23T07:09:26Z","failed_at":"e2e","signature_id":"0984e8df881596c9b834e29d1fcae2818b2d904758affc92d244ae142cb90020","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001062200\u003e: \n    failed to create NodePool np-hdmwqd: failed waiting for nodepool=\"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\" in resourcegroup=\"rg-candidate-4-20-grr9djf6qtvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"name\": \"c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T08:27:34.915224416Z\",\n      \"endTime\": \"2026-07-23T08:46:40.245736142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001062200\u003e: \n    failed to create NodePool np-hdmwqd: failed waiting for nodepool=\"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\" in resourcegroup=\"rg-candidate-4-20-grr9djf6qtvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"name\": \"c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T08:27:34.915224416Z\",\n      \"endTime\": \"2026-07-23T08:46:40.245736142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080132612415819776","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T03:27:16Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4d3b148439b4df4f52d47d1b2548396d727d3a035a9b9c57f367ece2e7dd585f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080132612415819776","occurred_at":"2026-07-23T03:27:16Z","failed_at":"e2e","signature_id":"55243e92d198af4223d7533882b14acb2a62c7ed804ca25f81a7e1527b9f37e6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0029b0ac0\u003e: \n    failed to create NodePool np-hl8829: failed waiting for nodepool=\"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\" in resourcegroup=\"rg-candidate-4-21-tv28gtjd9f9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"name\": \"d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T04:16:36.200454364Z\",\n      \"endTime\": \"2026-07-23T04:35:50.806314695Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0029b0ac0\u003e: \n    failed to create NodePool np-hl8829: failed waiting for nodepool=\"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\" in resourcegroup=\"rg-candidate-4-21-tv28gtjd9f9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"name\": \"d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T04:16:36.200454364Z\",\n      \"endTime\": \"2026-07-23T04:35:50.806314695Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2080110844716257280","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T02:00:46Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4d68ccb5a717a15dc4385fc40e17ea972f411c218a36d06a4972dcde15bd121b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2080110844716257280","occurred_at":"2026-07-23T02:00:46Z","failed_at":"e2e","signature_id":"61dff9a614aba5d6b259b363f11b1fe06cb1f30dcb1be30b825480e34108aa17","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-s9bzk7\" for cluster \"cluster-candidate-4-20-s9bzk7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010305a0\u003e: \n    failed to create NodePool np-s9bzk7: failed waiting for nodepool=\"np-s9bzk7\" for cluster \"cluster-candidate-4-20-s9bzk7\" in resourcegroup=\"rg-candidate-4-20-l7l7vvhqm2rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fabf9539-858a-402f-9f91-fefb74645581\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fabf9539-858a-402f-9f91-fefb74645581\",\n      \"name\": \"fabf9539-858a-402f-9f91-fefb74645581\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T02:18:43.115104063Z\",\n      \"endTime\": \"2026-07-23T02:37:48.305364094Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-s9bzk7\" for cluster \"cluster-candidate-4-20-s9bzk7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010305a0\u003e: \n    failed to create NodePool np-s9bzk7: failed waiting for nodepool=\"np-s9bzk7\" for cluster \"cluster-candidate-4-20-s9bzk7\" in resourcegroup=\"rg-candidate-4-20-l7l7vvhqm2rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fabf9539-858a-402f-9f91-fefb74645581\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fabf9539-858a-402f-9f91-fefb74645581\",\n      \"name\": \"fabf9539-858a-402f-9f91-fefb74645581\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T02:18:43.115104063Z\",\n      \"endTime\": \"2026-07-23T02:37:48.305364094Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6d23799176d69e078c369590e05c7788c2d63545ab5859634aa8fef547485019","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2080110844716257280","occurred_at":"2026-07-23T02:00:46Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080087615616323584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-23T00:28:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"94da7f4216999c0edac2d9f61d25bb8d38ae727c2b9b9a68333649a36a292727","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080087615616323584","occurred_at":"2026-07-23T00:28:28Z","failed_at":"e2e","signature_id":"1c30b2761273ca4099159b93719edbce4a961fd02fb2ee58f934ccaddd0614c0","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013a57d0\u003e: \n    add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013a57d0\u003e: \n    add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8223127140c833357088cdafd7290dc7db01297e09b7695d10e69e7ef3961f2","failure_pattern":"add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b8223127140c833357088cdafd7290dc7db01297e09b7695d10e69e7ef3961f2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080086441550942208","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-23T00:23:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}