{"meta":{"date":"2026-07-31","timezone":"UTC","generated_at":"2026-08-26T12:25:20Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":16,"failed_runs":10,"runs_with_occurrences":10,"runs_with_failure_pattern_matches":10,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T23:00:05Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"051a8d5323cc00be797b874ab28c0466d4338954008193904e2aa0af234b6d47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","failed_at":"e2e","signature_id":"4072a5302e0e12ad0124844ab5b66b17d835528aa52fe99adcf4fce54e5c47d5","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002930018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-x6fgs25whrrl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n \"name\": \"890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T23:22:53.3740224Z\",\n \"endTime\": \"2026-07-31T23:41:54.935883407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002930018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-x6fgs25whrrl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n \"name\": \"890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T23:22:53.3740224Z\",\n \"endTime\": \"2026-07-31T23:41:54.935883407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c41cfda35e14478cf4697fdfc69a721e02419d2f82ba42a13b5425c5d40a8e25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","failed_at":"e2e","signature_id":"b83962d63abd4751f1fb3741f6d0003b179d5f5f8ee9ba77b23446f84ffe47bf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-22-pb6jrkbs92lk/cluster-nightly-4-22-xssrv8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012790e0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-xssrv8: failed waiting for cluster=\"cluster-nightly-4-22-xssrv8\" in resourcegroup=\"rg-nightly-4-22-pb6jrkbs92lk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n \"name\": \"e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T23:32:11.571189856Z\",\n \"endTime\": \"2026-07-31T23:51:15.176646708Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-22-pb6jrkbs92lk/cluster-nightly-4-22-xssrv8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012790e0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-xssrv8: failed waiting for cluster=\"cluster-nightly-4-22-xssrv8\" in resourcegroup=\"rg-nightly-4-22-pb6jrkbs92lk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n \"name\": \"e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T23:32:11.571189856Z\",\n \"endTime\": \"2026-07-31T23:51:15.176646708Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9b5bc3dcd2eb88a004108ed19631ab439841e3bdf65de895edf25d5b7506a18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","failed_at":"e2e","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083281652888375296","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-31T20:00:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083272009831157760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-31T19:22:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083270259220942848","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-31T19:15:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083216167891111936","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-31T15:40:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083216004514582528","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-31T15:39:34Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083208396000989184","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T15:09:20Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d4703f488ae5d957b1ccd7a908e6420c3caaada46a884efebd01201129066f74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083208396000989184","occurred_at":"2026-07-31T15:09:20Z","failed_at":"e2e","signature_id":"90d4fdfb969a770390c563f242d33f0daa3473e0b30ba9027c45b262814b26d7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bgmptg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8ca0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lmdkg.apps.aro.p1l0j8m7f2t6o2v.wzf5.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bgmptg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8ca0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lmdkg.apps.aro.p1l0j8m7f2t6o2v.wzf5.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T12:56:41Z"},"failed_at":["e2e"],"failed_test_count":17,"occurrences":[{"row_id":"190d038a94d36f63621f39be2d4cdb1020d21c04fb8003f7b62e17fa252ca6d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"607628fb1634fcb469dfa7b2aaf90c7408c8b6f018f4a45ddf020ec477f5845c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-dnfb8scz27dr/cluster-candidate-4-21-pqc4pq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880ac0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pqc4pq: failed waiting for cluster=\"cluster-candidate-4-21-pqc4pq\" in resourcegroup=\"rg-candidate-4-21-dnfb8scz27dr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/851c1de8-8f67-418b-b90f-e7742af5043d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/851c1de8-8f67-418b-b90f-e7742af5043d\",\n \"name\": \"851c1de8-8f67-418b-b90f-e7742af5043d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:20:02.785041069Z\",\n \"endTime\": \"2026-07-31T13:39:11.012497136Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-dnfb8scz27dr/cluster-candidate-4-21-pqc4pq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880ac0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pqc4pq: failed waiting for cluster=\"cluster-candidate-4-21-pqc4pq\" in resourcegroup=\"rg-candidate-4-21-dnfb8scz27dr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/851c1de8-8f67-418b-b90f-e7742af5043d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/851c1de8-8f67-418b-b90f-e7742af5043d\",\n \"name\": \"851c1de8-8f67-418b-b90f-e7742af5043d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:20:02.785041069Z\",\n \"endTime\": \"2026-07-31T13:39:11.012497136Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2ed69e4c70f656ee5f1135d46b1fbafc7f60345d0ca65b0f77778b982b8cfee6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"320640368bf5c83bfbba9bfd950314011331135dd4a6ed380c39df80f1875f89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"91f19c9c4d341e3acfff76110011b76f24e350c1cb330ab3444ebcd6bbfd552d","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00085e420\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fqbpthjjl55h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/156fe0c6-70b3-44ca-a24a-1c027d41bc81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/156fe0c6-70b3-44ca-a24a-1c027d41bc81\",\n \"name\": \"156fe0c6-70b3-44ca-a24a-1c027d41bc81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:57.638510588Z\",\n \"endTime\": \"2026-07-31T13:36:11.103957595Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00085e420\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fqbpthjjl55h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/156fe0c6-70b3-44ca-a24a-1c027d41bc81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/156fe0c6-70b3-44ca-a24a-1c027d41bc81\",\n \"name\": \"156fe0c6-70b3-44ca-a24a-1c027d41bc81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:57.638510588Z\",\n \"endTime\": \"2026-07-31T13:36:11.103957595Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4150e719b550dbe18e6701b24e1377666fdc5c615e850d7210417c62979f1cd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"96ea83ea0690b8b14850b079a8cba0997c7b78b48c76c1ba7cafc61eb0bed221","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5e620\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mjk9zwq4gcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/ca6fb1ae-e7e6-4c18-85f7-6a10b4cd208b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/ca6fb1ae-e7e6-4c18-85f7-6a10b4cd208b\",\n \"name\": \"ca6fb1ae-e7e6-4c18-85f7-6a10b4cd208b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:53.067175076Z\",\n \"endTime\": \"2026-07-31T13:36:01.056235262Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5e620\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mjk9zwq4gcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/ca6fb1ae-e7e6-4c18-85f7-6a10b4cd208b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/ca6fb1ae-e7e6-4c18-85f7-6a10b4cd208b\",\n \"name\": \"ca6fb1ae-e7e6-4c18-85f7-6a10b4cd208b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:53.067175076Z\",\n \"endTime\": \"2026-07-31T13:36:01.056235262Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"57b8b734af0495f17e69df61414280e72d46a8790fc70889afd728f52271cdb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"88e1c6360611cb99db8664b8631b557fcf43a4ebd2fa58f417fa155ba9e0338f","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a986a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-4x7lzhfdfzv6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9dfa75cc-c808-41b5-be1b-39656aa5533f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9dfa75cc-c808-41b5-be1b-39656aa5533f\",\n \"name\": \"9dfa75cc-c808-41b5-be1b-39656aa5533f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:54.040864805Z\",\n \"endTime\": \"2026-07-31T13:36:01.003955846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a986a0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-4x7lzhfdfzv6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9dfa75cc-c808-41b5-be1b-39656aa5533f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9dfa75cc-c808-41b5-be1b-39656aa5533f\",\n \"name\": \"9dfa75cc-c808-41b5-be1b-39656aa5533f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:54.040864805Z\",\n \"endTime\": \"2026-07-31T13:36:01.003955846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"861959ed7d9236774ad6be0fa02d47a39ce26b2c475ddd98a4d84834af829ae8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"8a98a7e3baf3339560d27a9c17517801a79e44ba4e182a960f8845d43aa0cbd6","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a065e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-clfndjqd499c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7b094629-4083-493d-adde-c39585c46945\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7b094629-4083-493d-adde-c39585c46945\",\n \"name\": \"7b094629-4083-493d-adde-c39585c46945\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:57.023738163Z\",\n \"endTime\": \"2026-07-31T13:36:11.022032968Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a065e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-clfndjqd499c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7b094629-4083-493d-adde-c39585c46945\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7b094629-4083-493d-adde-c39585c46945\",\n \"name\": \"7b094629-4083-493d-adde-c39585c46945\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:57.023738163Z\",\n \"endTime\": \"2026-07-31T13:36:11.022032968Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f38bb22c18ad845c56f24354382d791acd3c620bf09b0c3f7519c42df62b186","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"dcf71a6679d8d1d79de65a03c8ed12db833efd96b553139459477d2e1cb8821f","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010026a0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-7qbshcklgtqf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5acc83f2-18e8-442a-add4-cfd69d5c4d0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5acc83f2-18e8-442a-add4-cfd69d5c4d0a\",\n \"name\": \"5acc83f2-18e8-442a-add4-cfd69d5c4d0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:55.565297549Z\",\n \"endTime\": \"2026-07-31T13:36:01.003665842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010026a0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-7qbshcklgtqf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5acc83f2-18e8-442a-add4-cfd69d5c4d0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5acc83f2-18e8-442a-add4-cfd69d5c4d0a\",\n \"name\": \"5acc83f2-18e8-442a-add4-cfd69d5c4d0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:55.565297549Z\",\n \"endTime\": \"2026-07-31T13:36:01.003665842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a4a51184d4cb1cb9d6c538d474db441c1650d785d7b35c612dbe530bbd7f968b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"f29993a7dbfecfb6d4c14d8061f6f5b4f50535f04f03fd4995a208bc605cb2ee","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:98]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093c260\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4c6xztbbdtsc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/f92d2bd1-579a-491e-8e0b-a1b581f7caa4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/f92d2bd1-579a-491e-8e0b-a1b581f7caa4\",\n \"name\": \"f92d2bd1-579a-491e-8e0b-a1b581f7caa4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:09.290012643Z\",\n \"endTime\": \"2026-07-31T13:36:11.103001582Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:98]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093c260\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4c6xztbbdtsc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/f92d2bd1-579a-491e-8e0b-a1b581f7caa4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/f92d2bd1-579a-491e-8e0b-a1b581f7caa4\",\n \"name\": \"f92d2bd1-579a-491e-8e0b-a1b581f7caa4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:09.290012643Z\",\n \"endTime\": \"2026-07-31T13:36:11.103001582Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a507d8af2691541f8cc748629c1bdce0f0ff1e6d4d37b487633b9cf0a64df8a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"552f71aba079936d33b679345d987c6915e1f5d4c7080b6617700d16096fb85b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78720\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wmqz9jgvp4kr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n \"name\": \"c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:19:22.344406702Z\",\n \"endTime\": \"2026-07-31T13:38:31.033150806Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78720\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wmqz9jgvp4kr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n \"name\": \"c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:19:22.344406702Z\",\n \"endTime\": \"2026-07-31T13:38:31.033150806Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac3e2f0edb7614d514cead56769da06be66b8962bf25c57d5490e869dff6ab43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"5c7e72fb745c231eec7d48bab072bb96a7bdd874d667b8db0a72e1fe40e0714c","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011f0c90\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/41d1575c-1ad0-49c0-bae2-6c5c16aa3b2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/41d1575c-1ad0-49c0-bae2-6c5c16aa3b2d\",\n \"name\": \"41d1575c-1ad0-49c0-bae2-6c5c16aa3b2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:18:10.012459965Z\",\n \"endTime\": \"2026-07-31T13:37:21.021390532Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011f0c90\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/41d1575c-1ad0-49c0-bae2-6c5c16aa3b2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/41d1575c-1ad0-49c0-bae2-6c5c16aa3b2d\",\n \"name\": \"41d1575c-1ad0-49c0-bae2-6c5c16aa3b2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:18:10.012459965Z\",\n \"endTime\": \"2026-07-31T13:37:21.021390532Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b14dfc4b619f2679153eb1feb2359842f77d41851fa4f39ab3df5eea05c05a59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"e766f3893d259999bf7feb58da887f8fddb7bf6407a6772cd14aea22320aa34c","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8a2a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m4rgz4zh45t4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n \"name\": \"69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:19:20.840846266Z\",\n \"endTime\": \"2026-07-31T13:38:21.001787978Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8a2a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m4rgz4zh45t4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n \"name\": \"69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:19:20.840846266Z\",\n \"endTime\": \"2026-07-31T13:38:21.001787978Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd3c6774ce92d581b82aaea223c77f1dc4172062bb15b78107f9dfa4a06b8d03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"e114cbedec1ecc31c1653acce737dc38649c0cf1338d14bda84b086679da4b71","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00220\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-kbbkwl845zdm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/dca71f39-d313-4d1d-9278-3f1acb295390\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/dca71f39-d313-4d1d-9278-3f1acb295390\",\n \"name\": \"dca71f39-d313-4d1d-9278-3f1acb295390\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:04.153079394Z\",\n \"endTime\": \"2026-07-31T13:36:10.998963651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00220\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-kbbkwl845zdm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/dca71f39-d313-4d1d-9278-3f1acb295390\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/dca71f39-d313-4d1d-9278-3f1acb295390\",\n \"name\": \"dca71f39-d313-4d1d-9278-3f1acb295390\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:04.153079394Z\",\n \"endTime\": \"2026-07-31T13:36:10.998963651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d012a8c33c409570f3c38104142b98fa805aa4e4ad01b31f5ae489af72551b18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"e27b57a2af9edacbd87acb6b031e968245cca218b2faada52de023bac352600d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"d0e63e099576bfd4d02da914e850e70f973afbd34ffb54f28fa5373c510d7c2f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eda220\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ffrbv6jnsvhd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/31975bc7-ff51-44b9-bc4f-f6b01aa7b10e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/31975bc7-ff51-44b9-bc4f-f6b01aa7b10e\",\n \"name\": \"31975bc7-ff51-44b9-bc4f-f6b01aa7b10e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:55.046866812Z\",\n \"endTime\": \"2026-07-31T13:36:11.08542774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eda220\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ffrbv6jnsvhd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/31975bc7-ff51-44b9-bc4f-f6b01aa7b10e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/31975bc7-ff51-44b9-bc4f-f6b01aa7b10e\",\n \"name\": \"31975bc7-ff51-44b9-bc4f-f6b01aa7b10e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:55.046866812Z\",\n \"endTime\": \"2026-07-31T13:36:11.08542774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e49a68d85ca353b415c9f3ca833480a7ab71be6f0eaee8770e0a69a19d01cbb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"e1b1ede7ff44850fbb3bb80eb24f477565a243eb8affcc66d3a502223e01b271","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:151]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dabc0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pzq4z2c4vrn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/a7ebc517-d8e2-49a5-8c90-b26287327010\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/a7ebc517-d8e2-49a5-8c90-b26287327010\",\n \"name\": \"a7ebc517-d8e2-49a5-8c90-b26287327010\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:52.926430897Z\",\n \"endTime\": \"2026-07-31T13:37:01.129081512Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:151]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010dabc0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pzq4z2c4vrn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/a7ebc517-d8e2-49a5-8c90-b26287327010\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/a7ebc517-d8e2-49a5-8c90-b26287327010\",\n \"name\": \"a7ebc517-d8e2-49a5-8c90-b26287327010\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:52.926430897Z\",\n \"endTime\": \"2026-07-31T13:37:01.129081512Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ef543afc06ca102c9ae608ea03dc0f117d50a6945a8263ce769869bab59bf591","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"0c1126e6d75cdc7e020f2cb29bb01e8e96f7e0b2735c876f1534e5ec6be736d6","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:95]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001104360\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-f7f6frqr9hfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n \"name\": \"1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:01.726559494Z\",\n \"endTime\": \"2026-07-31T13:36:11.083035507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:95]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001104360\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-f7f6frqr9hfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n \"name\": \"1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:17:01.726559494Z\",\n \"endTime\": \"2026-07-31T13:36:11.083035507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff172a6cb71eb1cb8cdbecb5943cafcc42f92e346309ea5759ee0eb1984407a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","failed_at":"e2e","signature_id":"7d25c20deea3d7f30cb4abd7abf0ed51fce890b92554c5eddf568175636f7041","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-vqgs79bkdksf/cluster-candidate-4-20-xprn42 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efa440\u003e: \n failed to create HCP cluster cluster-candidate-4-20-xprn42: failed waiting for cluster=\"cluster-candidate-4-20-xprn42\" in resourcegroup=\"rg-candidate-4-20-vqgs79bkdksf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/36fa5739-a6b1-49e9-92ba-01fbad5dfbcc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/36fa5739-a6b1-49e9-92ba-01fbad5dfbcc\",\n \"name\": \"36fa5739-a6b1-49e9-92ba-01fbad5dfbcc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:55.237884499Z\",\n \"endTime\": \"2026-07-31T13:36:11.102733678Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-vqgs79bkdksf/cluster-candidate-4-20-xprn42 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efa440\u003e: \n failed to create HCP cluster cluster-candidate-4-20-xprn42: failed waiting for cluster=\"cluster-candidate-4-20-xprn42\" in resourcegroup=\"rg-candidate-4-20-vqgs79bkdksf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/36fa5739-a6b1-49e9-92ba-01fbad5dfbcc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/36fa5739-a6b1-49e9-92ba-01fbad5dfbcc\",\n \"name\": \"36fa5739-a6b1-49e9-92ba-01fbad5dfbcc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:16:55.237884499Z\",\n \"endTime\": \"2026-07-31T13:36:11.102733678Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083172724083003392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-31T12:47:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083172285048426496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T12:45:50Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e37356cba51255237243236e90ce38a6382c2d3870cc7cf6ad35dccfac769c97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083172285048426496","occurred_at":"2026-07-31T12:45:50Z","failed_at":"e2e","signature_id":"f3674ccfd569851653672f515b42e2695a6f9f20f7a1fdd0153289b8e1933ea3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-n7l2w2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a001a0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-hhdrv.apps.aro.n7l8a2p4i7p3w7g.8d7k.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-n7l2w2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a001a0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-hhdrv.apps.aro.n7l8a2p4i7p3w7g.8d7k.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T11:56:32Z"},"failed_at":["e2e"],"failed_test_count":40,"occurrences":[{"row_id":"0609cbfb1c61f31159e3ed5768f4896c70a60be820c4f357b2d529d34b38c2f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"154f5ce0edd523ca30c455a745a6548ec024a3fecbf65bf5c3ed075cbcbefded","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8ed40\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hfwr6s9d22sj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bccc681f-f6ac-4411-9d61-3ac269e0f765\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bccc681f-f6ac-4411-9d61-3ac269e0f765\",\n \"name\": \"bccc681f-f6ac-4411-9d61-3ac269e0f765\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:48.450598357Z\",\n \"endTime\": \"2026-07-31T12:20:54.270198143Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8ed40\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hfwr6s9d22sj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bccc681f-f6ac-4411-9d61-3ac269e0f765\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bccc681f-f6ac-4411-9d61-3ac269e0f765\",\n \"name\": \"bccc681f-f6ac-4411-9d61-3ac269e0f765\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:48.450598357Z\",\n \"endTime\": \"2026-07-31T12:20:54.270198143Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0f9511ef6c929fddbc8c5611389d5d6e57d7861d8b9bcbbc6f2e79c0679591af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"a9fb25a5b04389ff66308052341420205d3ef1800d543533ae64ea0e425d0c94","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f852e0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fghb65pkrst5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c9d3890d-7986-4bb6-944f-22df746e8f43\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c9d3890d-7986-4bb6-944f-22df746e8f43\",\n \"name\": \"c9d3890d-7986-4bb6-944f-22df746e8f43\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:19.357855432Z\",\n \"endTime\": \"2026-07-31T13:06:35.420425018Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f852e0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fghb65pkrst5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c9d3890d-7986-4bb6-944f-22df746e8f43\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c9d3890d-7986-4bb6-944f-22df746e8f43\",\n \"name\": \"c9d3890d-7986-4bb6-944f-22df746e8f43\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:19.357855432Z\",\n \"endTime\": \"2026-07-31T13:06:35.420425018Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"133ab56cbfe1224243ea742046328fe36c2480eb4a4b875201a8db7d59d7e254","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"906befff4829c721784a271a1ed5b734525fa389cc169bbe54a38a5582736ac2","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-z5m5dr\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f843e0\u003e: \n failed to create HCP cluster np-noedge-z5m5dr: failed waiting for cluster=\"np-noedge-z5m5dr\" in resourcegroup=\"rg-np-noedge-z5m5dr-4pgr2zcl7d75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99a7ed5b-97f7-4f4d-89cc-3e575d210ed3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99a7ed5b-97f7-4f4d-89cc-3e575d210ed3\",\n \"name\": \"99a7ed5b-97f7-4f4d-89cc-3e575d210ed3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:49:34.935001181Z\",\n \"endTime\": \"2026-07-31T13:08:35.403292393Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-z5m5dr\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f843e0\u003e: \n failed to create HCP cluster np-noedge-z5m5dr: failed waiting for cluster=\"np-noedge-z5m5dr\" in resourcegroup=\"rg-np-noedge-z5m5dr-4pgr2zcl7d75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99a7ed5b-97f7-4f4d-89cc-3e575d210ed3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99a7ed5b-97f7-4f4d-89cc-3e575d210ed3\",\n \"name\": \"99a7ed5b-97f7-4f4d-89cc-3e575d210ed3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:49:34.935001181Z\",\n \"endTime\": \"2026-07-31T13:08:35.403292393Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"29c47e3d6a40b5ef441fd5b8748e75bfa5b02c2d840db10d42b4a466f2ec173c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"b36413c750154c83c94934fe75957298ae57383c5b39c72e5b0211ef615cf651","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018f2990\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b738141e-3370-4a09-858d-4ba4579c6392\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b738141e-3370-4a09-858d-4ba4579c6392\",\n \"name\": \"b738141e-3370-4a09-858d-4ba4579c6392\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:02:42.098678503Z\",\n \"endTime\": \"2026-07-31T12:21:44.482109342Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018f2990\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b738141e-3370-4a09-858d-4ba4579c6392\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b738141e-3370-4a09-858d-4ba4579c6392\",\n \"name\": \"b738141e-3370-4a09-858d-4ba4579c6392\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:02:42.098678503Z\",\n \"endTime\": \"2026-07-31T12:21:44.482109342Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2fca8bcef2a30c7ef600bdf897435fbb40d54b2aa256ae13de887cc00bc63ce2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"573385bd853245689458f88b5830b3711397b2693998619d3957c84c8fcd9eba","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-2rnnlx\" with version 4.22.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ae720\u003e: \n failed to create HCP cluster cluster-zstream-4-22-2rnnlx: failed waiting for cluster=\"cluster-zstream-4-22-2rnnlx\" in resourcegroup=\"rg-zstream-upgrade-4-22-vvtb4d8942ll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43775ced-61c9-4f11-adb9-c02152d8a88e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43775ced-61c9-4f11-adb9-c02152d8a88e\",\n \"name\": \"43775ced-61c9-4f11-adb9-c02152d8a88e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:52.729883882Z\",\n \"endTime\": \"2026-07-31T13:05:55.6469295Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-2rnnlx\" with version 4.22.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ae720\u003e: \n failed to create HCP cluster cluster-zstream-4-22-2rnnlx: failed waiting for cluster=\"cluster-zstream-4-22-2rnnlx\" in resourcegroup=\"rg-zstream-upgrade-4-22-vvtb4d8942ll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43775ced-61c9-4f11-adb9-c02152d8a88e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43775ced-61c9-4f11-adb9-c02152d8a88e\",\n \"name\": \"43775ced-61c9-4f11-adb9-c02152d8a88e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:52.729883882Z\",\n \"endTime\": \"2026-07-31T13:05:55.6469295Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"38ec93e8003ff5f1e548f13e2a393024400e9fea02fca93dd637e60229a85dc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"9cf910a3bca8d2d36ad62cb309495e5141ee82a6e645bd084cc851c4d118b418","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e1440\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-cpljbgvkgf69\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dfb8fd34-477f-4ced-b803-1c42007aa401\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dfb8fd34-477f-4ced-b803-1c42007aa401\",\n \"name\": \"dfb8fd34-477f-4ced-b803-1c42007aa401\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.959883097Z\",\n \"endTime\": \"2026-07-31T12:19:04.03877075Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e1440\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-cpljbgvkgf69\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dfb8fd34-477f-4ced-b803-1c42007aa401\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/dfb8fd34-477f-4ced-b803-1c42007aa401\",\n \"name\": \"dfb8fd34-477f-4ced-b803-1c42007aa401\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.959883097Z\",\n \"endTime\": \"2026-07-31T12:19:04.03877075Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c9add40b5ffb32fb20b2abe55397287eb1892b7b5aa28435937d139773db564","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"5d3cd084884fae8cfd89c38eeadaecaa371799f087ea96cfc086030755e93359","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-m96tjq with version 4.21.26\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f06c00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-m96tjq: failed waiting for cluster=\"np-version-upgrade-cluster-m96tjq\" in resourcegroup=\"rg-np-version-upgrade-m96tjq-jfhkkgg9dphf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/aa0ef237-5cc0-4a00-a9e8-bdac7065be85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/aa0ef237-5cc0-4a00-a9e8-bdac7065be85\",\n \"name\": \"aa0ef237-5cc0-4a00-a9e8-bdac7065be85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:48:22.258166392Z\",\n \"endTime\": \"2026-07-31T13:07:25.426997518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-m96tjq with version 4.21.26\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f06c00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-m96tjq: failed waiting for cluster=\"np-version-upgrade-cluster-m96tjq\" in resourcegroup=\"rg-np-version-upgrade-m96tjq-jfhkkgg9dphf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/aa0ef237-5cc0-4a00-a9e8-bdac7065be85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/aa0ef237-5cc0-4a00-a9e8-bdac7065be85\",\n \"name\": \"aa0ef237-5cc0-4a00-a9e8-bdac7065be85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:48:22.258166392Z\",\n \"endTime\": \"2026-07-31T13:07:25.426997518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42aa7bbe243e35b016d5d566a8609cf9fc168cc56b3058e21ed142e7fe1a4448","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"ea640be7c764230e5b99d89aa2b77d3f2b6d8bb8d7ff99fafcd3dd5fb29d6f10","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb2720\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-jsppkbsmcx8c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a22b49-2ce6-4baf-81da-d22f5a0174cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a22b49-2ce6-4baf-81da-d22f5a0174cd\",\n \"name\": \"94a22b49-2ce6-4baf-81da-d22f5a0174cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:05.396257205Z\",\n \"endTime\": \"2026-07-31T12:19:23.919438706Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb2720\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-jsppkbsmcx8c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a22b49-2ce6-4baf-81da-d22f5a0174cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a22b49-2ce6-4baf-81da-d22f5a0174cd\",\n \"name\": \"94a22b49-2ce6-4baf-81da-d22f5a0174cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:05.396257205Z\",\n \"endTime\": \"2026-07-31T12:19:23.919438706Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"441ffe1702990dda0291f5dab1794b7790469e0160326ad843e58efbf13e65f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"4e9036b6f419e800e27e7212dc4f4e9b7a7ecba2dc8927e0c7f9ea029ca257c3","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ae720\u003e: \n failed to create HCP cluster ea-list2cccc4: failed waiting for cluster=\"ea-list2cccc4\" in resourcegroup=\"ea-list-x46kchhvf45s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ee52c9f-627b-4a5d-9bb4-f4c4bba997ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ee52c9f-627b-4a5d-9bb4-f4c4bba997ab\",\n \"name\": \"0ee52c9f-627b-4a5d-9bb4-f4c4bba997ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:19.772087291Z\",\n \"endTime\": \"2026-07-31T13:06:25.406426295Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ae720\u003e: \n failed to create HCP cluster ea-list2cccc4: failed waiting for cluster=\"ea-list2cccc4\" in resourcegroup=\"ea-list-x46kchhvf45s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ee52c9f-627b-4a5d-9bb4-f4c4bba997ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0ee52c9f-627b-4a5d-9bb4-f4c4bba997ab\",\n \"name\": \"0ee52c9f-627b-4a5d-9bb4-f4c4bba997ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:19.772087291Z\",\n \"endTime\": \"2026-07-31T13:06:25.406426295Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4ac4db98e17e79486ff597f6432506b17d3ab97cf160ffd93497c47864816f37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"621048a5aac4e75e14d77235942749138b536b5328379c9bef941f0c7f0665f1","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ad30b0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ad30b0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded"}},{"row_id":"4f5067160f1897d2b7fc2d291eceb4a48ad230623a6450f2465f2eb298ad9575","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"c16325fdee230d6022e4009b6044e73ca1e8a7c8744e735f11ddf531839c9f28","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a147c0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5s82nrfxs6t2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ffadf69-6ed0-42a1-9028-5492804270af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ffadf69-6ed0-42a1-9028-5492804270af\",\n \"name\": \"9ffadf69-6ed0-42a1-9028-5492804270af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.577967957Z\",\n \"endTime\": \"2026-07-31T12:19:04.090736407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a147c0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5s82nrfxs6t2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ffadf69-6ed0-42a1-9028-5492804270af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9ffadf69-6ed0-42a1-9028-5492804270af\",\n \"name\": \"9ffadf69-6ed0-42a1-9028-5492804270af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.577967957Z\",\n \"endTime\": \"2026-07-31T12:19:04.090736407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"56c55808c6a3246c17e96e734d034e771dcc81e47babf7234218c07b93fdc8cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"96b4d73842c958bf6629ec879c25fb07e4c7558543d52db28eb48e259ef8c13a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014b2160\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fns56rh2tzfg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/947166d0-eada-48f0-ace7-0d274d0e63cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/947166d0-eada-48f0-ace7-0d274d0e63cf\",\n \"name\": \"947166d0-eada-48f0-ace7-0d274d0e63cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.575227197Z\",\n \"endTime\": \"2026-07-31T12:19:13.910773743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014b2160\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fns56rh2tzfg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/947166d0-eada-48f0-ace7-0d274d0e63cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/947166d0-eada-48f0-ace7-0d274d0e63cf\",\n \"name\": \"947166d0-eada-48f0-ace7-0d274d0e63cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.575227197Z\",\n \"endTime\": \"2026-07-31T12:19:13.910773743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d2fd71352859b4e7fcb14a3c81b3d873721ef69a9c9378d3d2bd28864819354","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"25008d95567a88684a2401a50fa214f41e0b84e2d1dec9c6147209f98565d05a","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000629640\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7dp9nqkbwqkr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bfa71a4f-39af-4f04-bdd3-83cd5b95d56a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bfa71a4f-39af-4f04-bdd3-83cd5b95d56a\",\n \"name\": \"bfa71a4f-39af-4f04-bdd3-83cd5b95d56a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:09.726755667Z\",\n \"endTime\": \"2026-07-31T13:06:15.776149529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000629640\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7dp9nqkbwqkr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bfa71a4f-39af-4f04-bdd3-83cd5b95d56a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bfa71a4f-39af-4f04-bdd3-83cd5b95d56a\",\n \"name\": \"bfa71a4f-39af-4f04-bdd3-83cd5b95d56a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:09.726755667Z\",\n \"endTime\": \"2026-07-31T13:06:15.776149529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7a35490830a470d3c7cfb0eb67c3324a49d217c4dfd041328be60084c7786b8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"22943a83b5a807a6c6ffb231051379e8f3cfba5ae7dfbed35ebe9fc1cc75321e","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00116a440\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ts2p9kgg4fh2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac4748c1-6c74-44e0-b3b6-28284478f166\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac4748c1-6c74-44e0-b3b6-28284478f166\",\n \"name\": \"ac4748c1-6c74-44e0-b3b6-28284478f166\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:49:39.849882975Z\",\n \"endTime\": \"2026-07-31T13:08:55.444933178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00116a440\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ts2p9kgg4fh2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac4748c1-6c74-44e0-b3b6-28284478f166\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac4748c1-6c74-44e0-b3b6-28284478f166\",\n \"name\": \"ac4748c1-6c74-44e0-b3b6-28284478f166\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:49:39.849882975Z\",\n \"endTime\": \"2026-07-31T13:08:55.444933178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7b2380b9b3d01cdc59cb23e22933f7929ed10868ed63a9345f73a1d625605b0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"a0b9995e1016cea8831a9346afa65cb0de0ccb6dc63fd78bef195dc758065084","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-5k9n28g497ks/cluster-candidate-4-20-hjlsfb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81440\u003e: \n failed to create HCP cluster cluster-candidate-4-20-hjlsfb: failed waiting for cluster=\"cluster-candidate-4-20-hjlsfb\" in resourcegroup=\"rg-candidate-4-20-5k9n28g497ks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0b13846c-dd7d-4d4d-b4d1-137e0480f8ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0b13846c-dd7d-4d4d-b4d1-137e0480f8ac\",\n \"name\": \"0b13846c-dd7d-4d4d-b4d1-137e0480f8ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:26.462893569Z\",\n \"endTime\": \"2026-07-31T12:19:44.259883588Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-5k9n28g497ks/cluster-candidate-4-20-hjlsfb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81440\u003e: \n failed to create HCP cluster cluster-candidate-4-20-hjlsfb: failed waiting for cluster=\"cluster-candidate-4-20-hjlsfb\" in resourcegroup=\"rg-candidate-4-20-5k9n28g497ks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0b13846c-dd7d-4d4d-b4d1-137e0480f8ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0b13846c-dd7d-4d4d-b4d1-137e0480f8ac\",\n \"name\": \"0b13846c-dd7d-4d4d-b4d1-137e0480f8ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:26.462893569Z\",\n \"endTime\": \"2026-07-31T12:19:44.259883588Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e2e338a628365a3de13a0579ac37c77df4f83646e8d90d75bd62d6bb98776f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"54c200ddcc4b6b4d6b780b2d6fedb782f78d76edab7742cd13cf4c42db5b6346","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-x49mzn\" with version 4.21.25 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a900\u003e: \n failed to create HCP cluster cluster-zstream-4-21-x49mzn: failed waiting for cluster=\"cluster-zstream-4-21-x49mzn\" in resourcegroup=\"rg-zstream-upgrade-4-21-s6pb7lc259xz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1969d982-18df-474c-9d4e-8f34126f20b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1969d982-18df-474c-9d4e-8f34126f20b6\",\n \"name\": \"1969d982-18df-474c-9d4e-8f34126f20b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:25.324169683Z\",\n \"endTime\": \"2026-07-31T13:06:35.445361278Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-x49mzn\" with version 4.21.25 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a900\u003e: \n failed to create HCP cluster cluster-zstream-4-21-x49mzn: failed waiting for cluster=\"cluster-zstream-4-21-x49mzn\" in resourcegroup=\"rg-zstream-upgrade-4-21-s6pb7lc259xz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1969d982-18df-474c-9d4e-8f34126f20b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1969d982-18df-474c-9d4e-8f34126f20b6\",\n \"name\": \"1969d982-18df-474c-9d4e-8f34126f20b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:25.324169683Z\",\n \"endTime\": \"2026-07-31T13:06:35.445361278Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e4f87ad3e7099addcf88d17f10b68e8feebf6b25522ca8376140b8135c136f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"8e0c318692726823fcb9ce05546450e7631bf82c6183481bfc41b70b98a6c9c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"723f7b27a9cb71599870027d12663525adfe554e9b07936872529641bf13f522","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pqzcmf with version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000da2f60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-pqzcmf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: failed waiting for cluster=\"np-version-upgrade-cluster-pqzcmf\" in resourcegroup=\"rg-np-version-upgrade-pqzcmf-mg822cfcxtc6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pqzcmf with version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000da2f60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-pqzcmf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: failed waiting for cluster=\"np-version-upgrade-cluster-pqzcmf\" in resourcegroup=\"rg-np-version-upgrade-pqzcmf-mg822cfcxtc6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded"}},{"row_id":"9bae8e9fda2595120dbb528e8d90024fafab6c09ee281bd66204af4ae14871c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"9e85a82c010fa440cf3284a6c4fbfbfb9628a5b3223a8d1335d2122dd79a4f90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"6d691098b25b913bc371e403e0b7755feb4b06711ad5a28796adcc05df7150c9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-21-bl96htjqcf4g/cluster-candidate-4-21-hj446k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009006e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-hj446k: failed waiting for cluster=\"cluster-candidate-4-21-hj446k\" in resourcegroup=\"rg-candidate-4-21-bl96htjqcf4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd436bf9-fa39-4c74-be2c-19031b5d8a32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd436bf9-fa39-4c74-be2c-19031b5d8a32\",\n \"name\": \"cd436bf9-fa39-4c74-be2c-19031b5d8a32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:03:41.603615391Z\",\n \"endTime\": \"2026-07-31T12:22:54.278721693Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-21-bl96htjqcf4g/cluster-candidate-4-21-hj446k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009006e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-hj446k: failed waiting for cluster=\"cluster-candidate-4-21-hj446k\" in resourcegroup=\"rg-candidate-4-21-bl96htjqcf4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd436bf9-fa39-4c74-be2c-19031b5d8a32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd436bf9-fa39-4c74-be2c-19031b5d8a32\",\n \"name\": \"cd436bf9-fa39-4c74-be2c-19031b5d8a32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:03:41.603615391Z\",\n \"endTime\": \"2026-07-31T12:22:54.278721693Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a459e346d956a9006ce1b404fa8511f401a929aee1fec924dfe3c83f8c112184","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"6ca1e397f6cd8b74e21110b1f5fbce7cdf75978c696708a7a4e2f0d0f28c4eec","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000837820\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t85w8t8785mc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b581b4ac-1d15-4534-82b2-e07c697e129d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b581b4ac-1d15-4534-82b2-e07c697e129d\",\n \"name\": \"b581b4ac-1d15-4534-82b2-e07c697e129d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:24.517393922Z\",\n \"endTime\": \"2026-07-31T12:20:24.47482417Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000837820\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t85w8t8785mc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b581b4ac-1d15-4534-82b2-e07c697e129d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b581b4ac-1d15-4534-82b2-e07c697e129d\",\n \"name\": \"b581b4ac-1d15-4534-82b2-e07c697e129d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:24.517393922Z\",\n \"endTime\": \"2026-07-31T12:20:24.47482417Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad80ce4f6b67083491c6ad458676e1c2474569643f23e6e841d0cf4cc5458b3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"dc6d73660e247263007affc3f4f8bb6b1eee9c82f5b3b07f0613d0203f757e79","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-9v4dmj\" with install version 4.20.31\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e40460\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-9v4dmj: failed waiting for cluster=\"cp-ystream-upgrade-4-21-9v4dmj\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-vh58khh8vnhw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/baf1756d-8268-4df6-9e23-e1ded22dfc81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/baf1756d-8268-4df6-9e23-e1ded22dfc81\",\n \"name\": \"baf1756d-8268-4df6-9e23-e1ded22dfc81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:23.334644442Z\",\n \"endTime\": \"2026-07-31T13:05:25.422674786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-9v4dmj\" with install version 4.20.31\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e40460\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-9v4dmj: failed waiting for cluster=\"cp-ystream-upgrade-4-21-9v4dmj\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-vh58khh8vnhw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/baf1756d-8268-4df6-9e23-e1ded22dfc81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/baf1756d-8268-4df6-9e23-e1ded22dfc81\",\n \"name\": \"baf1756d-8268-4df6-9e23-e1ded22dfc81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:23.334644442Z\",\n \"endTime\": \"2026-07-31T13:05:25.422674786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b039ed148b8bf9e402548a25ce772c74e17a4da8992727503fa791496b4d61be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"598d8eb936530f403448bb85cf6470e0108e2d0887675e2c547e501dd8e2698a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c86480\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4x6v64pzl4b7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/974dcfda-9d46-427e-b818-259c08234ede\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/974dcfda-9d46-427e-b818-259c08234ede\",\n \"name\": \"974dcfda-9d46-427e-b818-259c08234ede\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:53.95760242Z\",\n \"endTime\": \"2026-07-31T13:06:55.651631317Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c86480\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4x6v64pzl4b7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/974dcfda-9d46-427e-b818-259c08234ede\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/974dcfda-9d46-427e-b818-259c08234ede\",\n \"name\": \"974dcfda-9d46-427e-b818-259c08234ede\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:53.95760242Z\",\n \"endTime\": \"2026-07-31T13:06:55.651631317Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b39cbcb0966e03b3e23163c17e7dde745839471f721f6c73808f27a0ce8c192f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"adcb8d217761898b3931e51c0af842b6f345b29c2fc2fc186b0c88e3d636f4d6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-sjfwqd\" with version 4.20.30 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bd7020\u003e: \n failed to create HCP cluster cluster-zstream-4-20-sjfwqd: failed waiting for cluster=\"cluster-zstream-4-20-sjfwqd\" in resourcegroup=\"rg-zstream-upgrade-4-20-q9p5ns5d527f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0bc18e9d-bcc1-41f6-b1cf-4f6e60d95fab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0bc18e9d-bcc1-41f6-b1cf-4f6e60d95fab\",\n \"name\": \"0bc18e9d-bcc1-41f6-b1cf-4f6e60d95fab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:04.823163405Z\",\n \"endTime\": \"2026-07-31T13:05:05.806465578Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-sjfwqd\" with version 4.20.30 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bd7020\u003e: \n failed to create HCP cluster cluster-zstream-4-20-sjfwqd: failed waiting for cluster=\"cluster-zstream-4-20-sjfwqd\" in resourcegroup=\"rg-zstream-upgrade-4-20-q9p5ns5d527f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0bc18e9d-bcc1-41f6-b1cf-4f6e60d95fab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0bc18e9d-bcc1-41f6-b1cf-4f6e60d95fab\",\n \"name\": \"0bc18e9d-bcc1-41f6-b1cf-4f6e60d95fab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:04.823163405Z\",\n \"endTime\": \"2026-07-31T13:05:05.806465578Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b4c50306eaa93b5a70a265e1e86b022765a22954f1706ed102953c0c02b56b5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"a1a8244f248eb61e1f2faf3baa67ca04b0bc9681b815e51682f66356757489ea","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:87]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da5420\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x96nk5t52lqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d8349da-ca65-4b25-ab2c-8a2d9a3c4194\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d8349da-ca65-4b25-ab2c-8a2d9a3c4194\",\n \"name\": \"8d8349da-ca65-4b25-ab2c-8a2d9a3c4194\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.848279372Z\",\n \"endTime\": \"2026-07-31T12:19:04.109356555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:87]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da5420\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x96nk5t52lqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d8349da-ca65-4b25-ab2c-8a2d9a3c4194\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8d8349da-ca65-4b25-ab2c-8a2d9a3c4194\",\n \"name\": \"8d8349da-ca65-4b25-ab2c-8a2d9a3c4194\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.848279372Z\",\n \"endTime\": \"2026-07-31T12:19:04.109356555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b727b1a39ed77661dbbdc6a91cf719e146f046de24396974917b7d9f91b97851","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"b01e75249b24e2d6fa4aa72c2300fde23de6d71f37b3e421979785371f7df3b9","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf2660\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-zbvwjpmg5zsj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b4df381-d3c0-475b-a971-14022508b4d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b4df381-d3c0-475b-a971-14022508b4d8\",\n \"name\": \"1b4df381-d3c0-475b-a971-14022508b4d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:42.684138138Z\",\n \"endTime\": \"2026-07-31T13:06:55.387102153Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf2660\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-zbvwjpmg5zsj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b4df381-d3c0-475b-a971-14022508b4d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1b4df381-d3c0-475b-a971-14022508b4d8\",\n \"name\": \"1b4df381-d3c0-475b-a971-14022508b4d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:42.684138138Z\",\n \"endTime\": \"2026-07-31T13:06:55.387102153Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba259f2b23081f810ae6bd71e30a3a37475e95fcc03fed9b5eb3611e070336f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"eeb9e5277312013361542a3d83aa316edadd54874d2482f1d922aa82a8006029","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b126a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-sz49bv6rn2fk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8706ec1c-f58d-4387-8b21-63b285fa6ab8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8706ec1c-f58d-4387-8b21-63b285fa6ab8\",\n \"name\": \"8706ec1c-f58d-4387-8b21-63b285fa6ab8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:15.836681581Z\",\n \"endTime\": \"2026-07-31T12:20:24.452286649Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b126a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-sz49bv6rn2fk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8706ec1c-f58d-4387-8b21-63b285fa6ab8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8706ec1c-f58d-4387-8b21-63b285fa6ab8\",\n \"name\": \"8706ec1c-f58d-4387-8b21-63b285fa6ab8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:15.836681581Z\",\n \"endTime\": \"2026-07-31T12:20:24.452286649Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c661660793534cbbc85c1e6630fca0e44d44ffb221e1200e946a38af5f991a32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"aae7e95022d6cbcd106a7aee2159322a5ee8fa7942d388b8298ca95b862dee37","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008811a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-6x4cp4vz882r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/880e1c6c-93e7-4bd6-a31d-235bdcf0a976\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/880e1c6c-93e7-4bd6-a31d-235bdcf0a976\",\n \"name\": \"880e1c6c-93e7-4bd6-a31d-235bdcf0a976\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:03:05.508336621Z\",\n \"endTime\": \"2026-07-31T12:22:14.267159647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008811a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-6x4cp4vz882r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/880e1c6c-93e7-4bd6-a31d-235bdcf0a976\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/880e1c6c-93e7-4bd6-a31d-235bdcf0a976\",\n \"name\": \"880e1c6c-93e7-4bd6-a31d-235bdcf0a976\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:03:05.508336621Z\",\n \"endTime\": \"2026-07-31T12:22:14.267159647Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c8c6bf4d4599dd4c7026f86d9944d248f18953fc4a094dbfb49e0b023a6d693e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"e33d17059d28a9d326e9f2de865ed6ddad82a5c325137c25ff2d33a25a8cfff9","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072ec20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fbrwp6lcxq4w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/04d03637-62f4-4046-85f8-14be1f163c67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/04d03637-62f4-4046-85f8-14be1f163c67\",\n \"name\": \"04d03637-62f4-4046-85f8-14be1f163c67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:56.552833709Z\",\n \"endTime\": \"2026-07-31T12:20:04.295628961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072ec20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fbrwp6lcxq4w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/04d03637-62f4-4046-85f8-14be1f163c67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/04d03637-62f4-4046-85f8-14be1f163c67\",\n \"name\": \"04d03637-62f4-4046-85f8-14be1f163c67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:56.552833709Z\",\n \"endTime\": \"2026-07-31T12:20:04.295628961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce68f55bf96aa9094f1b62116defb720b602368b41fd1ad530a9e9ff053b5f05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"2f877087d24f1e4e43127361049faeed95382712e0b72bf3191afadfb5f39315","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000626de0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qw9qszwqlt87\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8707b5e-5725-48cd-92ce-c702a23e8063\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8707b5e-5725-48cd-92ce-c702a23e8063\",\n \"name\": \"e8707b5e-5725-48cd-92ce-c702a23e8063\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:35:15.216018644Z\",\n \"endTime\": \"2026-07-31T13:54:18.177825714Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000626de0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qw9qszwqlt87\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8707b5e-5725-48cd-92ce-c702a23e8063\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8707b5e-5725-48cd-92ce-c702a23e8063\",\n \"name\": \"e8707b5e-5725-48cd-92ce-c702a23e8063\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:35:15.216018644Z\",\n \"endTime\": \"2026-07-31T13:54:18.177825714Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d286adea8d97b1dbc3ab069a5a9d1026d652650d52956144e598fa90053eb9a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"6c5c182039f7672dc075fa72b54b020a27dba8e597655cef85eec4043eabaa40","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010161c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cpcfj5mc25gd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b8f117d7-f14c-4061-9837-e9dee38d0c99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b8f117d7-f14c-4061-9837-e9dee38d0c99\",\n \"name\": \"b8f117d7-f14c-4061-9837-e9dee38d0c99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:11.058914297Z\",\n \"endTime\": \"2026-07-31T13:06:15.389856542Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010161c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cpcfj5mc25gd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b8f117d7-f14c-4061-9837-e9dee38d0c99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b8f117d7-f14c-4061-9837-e9dee38d0c99\",\n \"name\": \"b8f117d7-f14c-4061-9837-e9dee38d0c99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:11.058914297Z\",\n \"endTime\": \"2026-07-31T13:06:15.389856542Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e338509c93505577847891065fe935fe61046db58381ca76af8a6a39c2c4f290","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"cdf5f5f8f81b6af2194a31a0ee8b9bf30b4a9fa000facb1d01d67199637f56b4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-kdm2x5\" with version 4.19.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00087b3c0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-kdm2x5: failed waiting for cluster=\"cluster-zstream-4-19-kdm2x5\" in resourcegroup=\"rg-zstream-upgrade-4-19-x5j44b6n6jh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49f3b760-56d6-4bb8-b9ad-6d11be6b2365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49f3b760-56d6-4bb8-b9ad-6d11be6b2365\",\n \"name\": \"49f3b760-56d6-4bb8-b9ad-6d11be6b2365\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:14.967143155Z\",\n \"endTime\": \"2026-07-31T13:06:15.685755904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-kdm2x5\" with version 4.19.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00087b3c0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-kdm2x5: failed waiting for cluster=\"cluster-zstream-4-19-kdm2x5\" in resourcegroup=\"rg-zstream-upgrade-4-19-x5j44b6n6jh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49f3b760-56d6-4bb8-b9ad-6d11be6b2365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/49f3b760-56d6-4bb8-b9ad-6d11be6b2365\",\n \"name\": \"49f3b760-56d6-4bb8-b9ad-6d11be6b2365\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:47:14.967143155Z\",\n \"endTime\": \"2026-07-31T13:06:15.685755904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e650de107337f088f3dbaea0a0ace30b930c70f894c930cbb55a1f5afc27b4c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"42a109ad57658f9dd5b2ea83c3874cc37d40e669625c71bfdb720ca6a6021a88","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-6wkt8pflljjl/cluster-candidate-4-22-lmtk24 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001514800\u003e: \n failed to create HCP cluster cluster-candidate-4-22-lmtk24: failed waiting for cluster=\"cluster-candidate-4-22-lmtk24\" in resourcegroup=\"rg-candidate-4-22-6wkt8pflljjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n \"name\": \"0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:24.136498027Z\",\n \"endTime\": \"2026-07-31T13:05:25.610298911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-6wkt8pflljjl/cluster-candidate-4-22-lmtk24 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001514800\u003e: \n failed to create HCP cluster cluster-candidate-4-22-lmtk24: failed waiting for cluster=\"cluster-candidate-4-22-lmtk24\" in resourcegroup=\"rg-candidate-4-22-6wkt8pflljjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n \"name\": \"0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:24.136498027Z\",\n \"endTime\": \"2026-07-31T13:05:25.610298911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6869fcc8b02c54ffca2380a60f490cfa28e1a1a6805caf58d9343d662d44979","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"1effbddc31f6279209dedbf019d17f5e620f7b5d749048b6663ae3183edf1438","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df4740\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnfkplvhg68n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\",\n \"name\": \"3b133ef7-396b-4152-900c-02c4e236373d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:02:16.489433823Z\",\n \"endTime\": \"2026-07-31T12:21:24.280178661Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df4740\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnfkplvhg68n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\",\n \"name\": \"3b133ef7-396b-4152-900c-02c4e236373d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:02:16.489433823Z\",\n \"endTime\": \"2026-07-31T12:21:24.280178661Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"edac9cc381367af437ac29163f130ebd571fd223a59a63b78f761df3a8ffbb2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"f828b6cf6e4c8d20d69cac8a4dc69f9f0da2dbd01ee9086224275c63d1cafafc","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-8hxplg with version 4.21.26\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002fc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8hxplg: failed waiting for cluster=\"np-version-upgrade-cluster-8hxplg\" in resourcegroup=\"rg-np-version-upgrade-8hxplg-h9xspkzr9vjh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cf2825f5-95a5-44dc-bba0-514137a9699f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cf2825f5-95a5-44dc-bba0-514137a9699f\",\n \"name\": \"cf2825f5-95a5-44dc-bba0-514137a9699f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:48:27.900137778Z\",\n \"endTime\": \"2026-07-31T13:07:35.43155508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-8hxplg with version 4.21.26\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001002fc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8hxplg: failed waiting for cluster=\"np-version-upgrade-cluster-8hxplg\" in resourcegroup=\"rg-np-version-upgrade-8hxplg-h9xspkzr9vjh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cf2825f5-95a5-44dc-bba0-514137a9699f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cf2825f5-95a5-44dc-bba0-514137a9699f\",\n \"name\": \"cf2825f5-95a5-44dc-bba0-514137a9699f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:48:27.900137778Z\",\n \"endTime\": \"2026-07-31T13:07:35.43155508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee094f41b6fd9fac32cfa1daa03dbfef1c232fea87f362654ab2bf925214a1dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"e67e9411babb49a9fa9afa754913a479d3d1e68a9b54dca6fdd595642cb6d3c0","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db5b80\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hnpcczndnkbw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e1ae0714-ba0e-4356-85f2-f6dd33d6700b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e1ae0714-ba0e-4356-85f2-f6dd33d6700b\",\n \"name\": \"e1ae0714-ba0e-4356-85f2-f6dd33d6700b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:18.672452299Z\",\n \"endTime\": \"2026-07-31T12:20:24.434902164Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db5b80\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hnpcczndnkbw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e1ae0714-ba0e-4356-85f2-f6dd33d6700b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e1ae0714-ba0e-4356-85f2-f6dd33d6700b\",\n \"name\": \"e1ae0714-ba0e-4356-85f2-f6dd33d6700b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:01:18.672452299Z\",\n \"endTime\": \"2026-07-31T12:20:24.434902164Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f591185881f05eb5fdfc488a4da4b4344939b326e336e678a911e00c70666f55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"48aa84bf3a4ab3ae6e2edfbe0ed7326dbe11e7a129a51125ec3338ac1e04aa37","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000927000\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-rgpcvx8v4t8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ccc1f6c-590e-4616-b432-628b64781c42\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ccc1f6c-590e-4616-b432-628b64781c42\",\n \"name\": \"1ccc1f6c-590e-4616-b432-628b64781c42\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:34:08.360101146Z\",\n \"endTime\": \"2026-07-31T13:53:08.175169994Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000927000\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-rgpcvx8v4t8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ccc1f6c-590e-4616-b432-628b64781c42\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1ccc1f6c-590e-4616-b432-628b64781c42\",\n \"name\": \"1ccc1f6c-590e-4616-b432-628b64781c42\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:34:08.360101146Z\",\n \"endTime\": \"2026-07-31T13:53:08.175169994Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fe7e74cbf891d2350d58218fb1915c02082eb41e9d9ce9177861c469fd27fd01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"63334c83aea6313567e61807e669f6eaf93ce8a667efae52f6a8a32ea39a7356","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eef220\u003e: \n failed to create HCP cluster np-dg-minor-zqwbh4: failed waiting for cluster=\"np-dg-minor-zqwbh4\" in resourcegroup=\"rg-np-dg-minor-zqwbh4-r2fdwmzx4zm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad53a6d-269e-46e7-a059-ab6358cff0b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad53a6d-269e-46e7-a059-ab6358cff0b7\",\n \"name\": \"fad53a6d-269e-46e7-a059-ab6358cff0b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:33:30.414838835Z\",\n \"endTime\": \"2026-07-31T13:52:48.232175759Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eef220\u003e: \n failed to create HCP cluster np-dg-minor-zqwbh4: failed waiting for cluster=\"np-dg-minor-zqwbh4\" in resourcegroup=\"rg-np-dg-minor-zqwbh4-r2fdwmzx4zm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad53a6d-269e-46e7-a059-ab6358cff0b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fad53a6d-269e-46e7-a059-ab6358cff0b7\",\n \"name\": \"fad53a6d-269e-46e7-a059-ab6358cff0b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T13:33:30.414838835Z\",\n \"endTime\": \"2026-07-31T13:52:48.232175759Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff26fe4a12788d4aef1538331a9ceccc0bc19752cbd1b29aec8ca3a00fef265d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"7a9ddd001fe58609fa7bc919b8085974ddd5414eca89207c89a656fef92e9af2","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d53480\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2wrjhwjzfnm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/10e4be36-6a64-4cc5-ad9f-6444b283831d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/10e4be36-6a64-4cc5-ad9f-6444b283831d\",\n \"name\": \"10e4be36-6a64-4cc5-ad9f-6444b283831d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:42.555243039Z\",\n \"endTime\": \"2026-07-31T13:05:45.397710912Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d53480\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2wrjhwjzfnm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/10e4be36-6a64-4cc5-ad9f-6444b283831d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/10e4be36-6a64-4cc5-ad9f-6444b283831d\",\n \"name\": \"10e4be36-6a64-4cc5-ad9f-6444b283831d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:46:42.555243039Z\",\n \"endTime\": \"2026-07-31T13:05:45.397710912Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ffc634cfb981d0b419efd2a8389e3f188815a89ce87a6ab2680c8145ba61e88a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","failed_at":"e2e","signature_id":"89a89a3a29d9fb04451cf8ae9e348d1106823e4aa50b21a935ec61e75735db22","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e2d8a0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-tnjn9dftvqc6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e2e1115-17f3-4750-a23d-14ded7c0b21c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e2e1115-17f3-4750-a23d-14ded7c0b21c\",\n \"name\": \"5e2e1115-17f3-4750-a23d-14ded7c0b21c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.839546618Z\",\n \"endTime\": \"2026-07-31T12:19:04.076010137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e2d8a0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-tnjn9dftvqc6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e2e1115-17f3-4750-a23d-14ded7c0b21c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e2e1115-17f3-4750-a23d-14ded7c0b21c\",\n \"name\": \"5e2e1115-17f3-4750-a23d-14ded7c0b21c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T12:00:04.839546618Z\",\n \"endTime\": \"2026-07-31T12:19:04.076010137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":40,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":40,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083129615265566720","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T09:56:17Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"06d189bbce8a4bcbcdb5afc547dd4b343aa251874f50c40e9127593ce52ff26e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083129615265566720","occurred_at":"2026-07-31T09:56:17Z","failed_at":"e2e","signature_id":"0331cad4f2a58b011ffbb47661db7de7b69b057a6cf4c77587a1708b2b88ab94","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bfhpb4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84940\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fsb9t.apps.aro.m0y5v1t7g5f7f6f.6ugv.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bfhpb4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84940\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fsb9t.apps.aro.m0y5v1t7g5f7f6f.6ugv.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T09:29:33Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1b2632701adb3a8ab4d46339b810e16414da1543a2d116e5e2cf98f92f59b567","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","occurred_at":"2026-07-31T09:29:33Z","failed_at":"e2e","signature_id":"cccdad4e7c305a79b69a618a299a0aca191ce2b7e21021783762aa34e265d4b0","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000820940\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-kqrzwh8lm75q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5e5c2e4-dda7-4998-a4b0-0bb19c5affd0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5e5c2e4-dda7-4998-a4b0-0bb19c5affd0\",\n \"name\": \"b5e5c2e4-dda7-4998-a4b0-0bb19c5affd0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T10:36:08.84862202Z\",\n \"endTime\": \"2026-07-31T10:55:11.142526884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000820940\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-kqrzwh8lm75q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5e5c2e4-dda7-4998-a4b0-0bb19c5affd0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5e5c2e4-dda7-4998-a4b0-0bb19c5affd0\",\n \"name\": \"b5e5c2e4-dda7-4998-a4b0-0bb19c5affd0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T10:36:08.84862202Z\",\n \"endTime\": \"2026-07-31T10:55:11.142526884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d789b53d21eaaf930d1dfc75f39a59bcd411ba9676fd3229452222b6a4eda53e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","occurred_at":"2026-07-31T09:29:33Z","failed_at":"e2e","signature_id":"b0aeb13bb7636b812ad85a1503b71e1488603b540400913f781cd52a11d81b47","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00132e4a0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-r9jx8zhqzt6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e461518c-11cf-4aeb-812d-7f544d241420\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e461518c-11cf-4aeb-812d-7f544d241420\",\n \"name\": \"e461518c-11cf-4aeb-812d-7f544d241420\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T10:34:56.320100473Z\",\n \"endTime\": \"2026-07-31T10:54:01.240534228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00132e4a0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-r9jx8zhqzt6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e461518c-11cf-4aeb-812d-7f544d241420\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e461518c-11cf-4aeb-812d-7f544d241420\",\n \"name\": \"e461518c-11cf-4aeb-812d-7f544d241420\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-31T10:34:56.320100473Z\",\n \"endTime\": \"2026-07-31T10:54:01.240534228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083096246586707968","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T07:43:41Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"21ff79d647e6c52ab3cd24f939ce5b57f6352e09afcc626112e247155e498b60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083096246586707968","occurred_at":"2026-07-31T07:43:41Z","failed_at":"e2e","signature_id":"b3787027bfe54fcb1e26887ea60cdb16b0f24ce43553bbde8fc9ce9b610a6a9b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-4mfgjw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008bf000\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-bmrbj.apps.aro.o3t7k5g0z5t7x0f.bwkb.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-4mfgjw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008bf000\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-bmrbj.apps.aro.o3t7k5g0z5t7x0f.bwkb.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T06:26:10Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"66f6d870a4a0a2d8bb213b53a8581dedb9a33abb01631bca6ab2d834e86db259","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","failed_at":"e2e","signature_id":"423dbfb7ba3def13bdce0df4b3635387513bf45addadf90a3c38c3ed7dac329a","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n \u003c*errors.joinError | 0xc001e04108\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\", caused by: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n \u003c*errors.joinError | 0xc001e04108\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\", caused by: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded"}},{"row_id":"c3532ae2c9f6af060eabdbc912b9c5851ed0226f07a56dac5a0f662878f02cfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","failed_at":"e2e","signature_id":"31b06f078f50236696796c5d7f165517cbc573486d9f7fc2bf92807bf3599f7d","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ad1d0\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ad1d0\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded"}},{"row_id":"dfb76d88eb708efd605847492098b7c4298d5bc47cc15315f8fb9267202ba551","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","failed_at":"e2e","signature_id":"104e9b9a5b09b9083011ea98bb3e00c0b9a6c2cf07df0e0d78da950283d3b4ac","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n \u003c*errors.joinError | 0xc00137c558\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\", caused by: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n \u003c*errors.joinError | 0xc00137c558\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\", caused by: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083009755474825216","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-31T02:00:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e44f6befd2b514cc193789a632f0aff0511ca57944f011d3e527846a408f90fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083009755474825216","occurred_at":"2026-07-31T02:00:00Z","failed_at":"e2e","signature_id":"033938777cbf5c901d58fef72baead815b4f5bdf96d12c88b67aa03ba6ca5fdf","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093af20\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jtfcvfq7vbjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtfcvfq7vbjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: NoRegisteredProviderFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"NoRegisteredProviderFound\",\n \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093af20\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jtfcvfq7vbjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtfcvfq7vbjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: NoRegisteredProviderFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"NoRegisteredProviderFound\",\n \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"01f15bd280db5d89c938e3274f737786d9867d8b1ba2c2fbc749dba703089f61","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["01f15bd280db5d89c938e3274f737786d9867d8b1ba2c2fbc749dba703089f61"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}