{"meta":{"start_date":"2026-07-22","end_date":"2026-07-22","start_at":"2026-07-22T00:00:00Z","end_at":"2026-07-23T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T06:38:50Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":17,"failed_runs":10,"raw_occurrences":12,"matched_occurrences":12,"runs_affected":10},"rows":[{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":23.529411764705884,"anchor_occurrences":4,"after_last_push_count":7,"trend_counts":[0,0,0,0,1,2,4],"trend_range":"2026-07-16..2026-07-22 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080037924878946304","occurred_at":"2026-07-22T21:11:01Z","signature_id":"a8557b42abcd154be2dbc1d4e7697becab9013bfd159699fe454c3dbe99a6090","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079895431134842880","occurred_at":"2026-07-22T11:44:47Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079748452178202624","occurred_at":"2026-07-22T02:00:45Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079731450374000640","occurred_at":"2026-07-22T00:53:11Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":17.647058823529413,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,1,1,0,3],"trend_range":"2026-07-16..2026-07-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":5,"prior_last_seen_at":"2026-07-19T23:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-qcvwjv\" for cluster \"cluster-candidate-4-20-qcvwjv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0026a6940\u003e: \n    failed to create NodePool np-qcvwjv: failed waiting for nodepool=\"np-qcvwjv\" for cluster \"cluster-candidate-4-20-qcvwjv\" in resourcegroup=\"rg-candidate-4-20-kwkc79fpjrx2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/a9e953c4-caff-45c7-a41d-8606a89637ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/a9e953c4-caff-45c7-a41d-8606a89637ea\",\n      \"name\": \"a9e953c4-caff-45c7-a41d-8606a89637ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T22:01:39.504754038Z\",\n      \"endTime\": \"2026-07-22T22:20:44.648330077Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-qcvwjv\" for cluster \"cluster-candidate-4-20-qcvwjv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0026a6940\u003e: \n    failed to create NodePool np-qcvwjv: failed waiting for nodepool=\"np-qcvwjv\" for cluster \"cluster-candidate-4-20-qcvwjv\" in resourcegroup=\"rg-candidate-4-20-kwkc79fpjrx2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/a9e953c4-caff-45c7-a41d-8606a89637ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/a9e953c4-caff-45c7-a41d-8606a89637ea\",\n      \"name\": \"a9e953c4-caff-45c7-a41d-8606a89637ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T22:01:39.504754038Z\",\n      \"endTime\": \"2026-07-22T22:20:44.648330077Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-79gssd\" for cluster \"cluster-candidate-4-21-79gssd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00061b560\u003e: \n    failed to create NodePool np-79gssd: failed waiting for nodepool=\"np-79gssd\" for cluster \"cluster-candidate-4-21-79gssd\" in resourcegroup=\"rg-candidate-4-21-rjrdqb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/985baa3e-b3d4-4c63-a507-079bb53207c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/985baa3e-b3d4-4c63-a507-079bb53207c7\",\n      \"name\": \"985baa3e-b3d4-4c63-a507-079bb53207c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T06:00:51.385178815Z\",\n      \"endTime\": \"2026-07-22T06:19:51.057036582Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-79gssd\" for cluster \"cluster-candidate-4-21-79gssd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00061b560\u003e: \n    failed to create NodePool np-79gssd: failed waiting for nodepool=\"np-79gssd\" for cluster \"cluster-candidate-4-21-79gssd\" in resourcegroup=\"rg-candidate-4-21-rjrdqb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/985baa3e-b3d4-4c63-a507-079bb53207c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/985baa3e-b3d4-4c63-a507-079bb53207c7\",\n      \"name\": \"985baa3e-b3d4-4c63-a507-079bb53207c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T06:00:51.385178815Z\",\n      \"endTime\": \"2026-07-22T06:19:51.057036582Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-v8pbwf\" for cluster \"cluster-candidate-4-20-v8pbwf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb8780\u003e: \n    failed to create NodePool np-v8pbwf: failed waiting for nodepool=\"np-v8pbwf\" for cluster \"cluster-candidate-4-20-v8pbwf\" in resourcegroup=\"rg-candidate-4-20-tr95ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/4dcd2df8-598c-4ddd-ae99-b9917b436164\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/4dcd2df8-598c-4ddd-ae99-b9917b436164\",\n      \"name\": \"4dcd2df8-598c-4ddd-ae99-b9917b436164\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T01:51:53.026515054Z\",\n      \"endTime\": \"2026-07-22T02:11:00.801292673Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-v8pbwf\" for cluster \"cluster-candidate-4-20-v8pbwf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb8780\u003e: \n    failed to create NodePool np-v8pbwf: failed waiting for nodepool=\"np-v8pbwf\" for cluster \"cluster-candidate-4-20-v8pbwf\" in resourcegroup=\"rg-candidate-4-20-tr95ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/4dcd2df8-598c-4ddd-ae99-b9917b436164\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/4dcd2df8-598c-4ddd-ae99-b9917b436164\",\n      \"name\": \"4dcd2df8-598c-4ddd-ae99-b9917b436164\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T01:51:53.026515054Z\",\n      \"endTime\": \"2026-07-22T02:11:00.801292673Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080036190387441664","occurred_at":"2026-07-22T21:04:07Z","signature_id":"78a8d6258739afb8e12558c28fcccc4e14633dba9764e96e9ace0c147dbc3c51","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079792930498809856","occurred_at":"2026-07-22T04:57:29Z","signature_id":"9398019cf66f84e56213a10b2bdac60b520839a24df306fb10defedff1a34ec1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079730275666890752","occurred_at":"2026-07-22T00:48:31Z","signature_id":"296fba6b5630a0876c842ef157378bd0a4ac3e8b1a977f7b931413e0fc9f61f5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":2,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,8,2],"trend_range":"2026-07-16..2026-07-22 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005871c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-sj5lp4s8hdhx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"name\": \"913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:04:09.655224875Z\",\n      \"endTime\": \"2026-07-22T02:23:21.611762653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005871c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-sj5lp4s8hdhx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"name\": \"913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:04:09.655224875Z\",\n      \"endTime\": \"2026-07-22T02:23:21.611762653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-qnfrfst59rxk/cluster-candidate-4-20-z9f7qm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b10b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-z9f7qm: failed waiting for cluster=\"cluster-candidate-4-20-z9f7qm\" in resourcegroup=\"rg-candidate-4-20-qnfrfst59rxk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"name\": \"952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:05:40.15224731Z\",\n      \"endTime\": \"2026-07-22T02:24:41.285079074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-qnfrfst59rxk/cluster-candidate-4-20-z9f7qm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b10b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-z9f7qm: failed waiting for cluster=\"cluster-candidate-4-20-z9f7qm\" in resourcegroup=\"rg-candidate-4-20-qnfrfst59rxk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"name\": \"952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:05:40.15224731Z\",\n      \"endTime\": \"2026-07-22T02:24:41.285079074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079748452178202624","occurred_at":"2026-07-22T02:00:45Z","signature_id":"575d927936ae879900e3bbf127a88fd09b1860584bee57c70dfe366e81ab46e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079748452178202624","occurred_at":"2026-07-22T02:00:45Z","signature_id":"f25a33fd6dcb03a98d82676f51c425eade08f236835c100a86a21ec4ac380b42","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2a5131f222749c0c944dc75ca14b7d24e45ce57c0da62c7a9019b2ce22a3e31d","failure_pattern":"ERROR CODE: InternalServerError; detail code DenyAssignmentAuthorizationFailed; detail message Access denied by deny assignment for action \"Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action\".; provider Microsoft.Authorization","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-16..2026-07-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00003c078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xxxldl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"name\": \"9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T08:31:33.476055825Z\",\n      \"endTime\": \"2026-07-22T08:50:37.393817181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n      - VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00003c078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xxxldl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"name\": \"9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T08:31:33.476055825Z\",\n      \"endTime\": \"2026-07-22T08:50:37.393817181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n      - VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079841571435450368","occurred_at":"2026-07-22T08:10:44Z","signature_id":"ea30ba3347f94c61ae7d6827ed5775c854aeecaddab6076a77ef7240e95ba09b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":125,"trend_counts":[0,0,0,0,82,42,1],"trend_range":"2026-07-16..2026-07-22 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006e58a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rp6jsc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"name\": \"174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T12:40:43.885322535Z\",\n      \"endTime\": \"2026-07-22T12:59:47.104093689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006e58a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rp6jsc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"name\": \"174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T12:40:43.885322535Z\",\n      \"endTime\": \"2026-07-22T12:59:47.104093689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079895694151258112","occurred_at":"2026-07-22T11:45:50Z","signature_id":"8230da4579d8f1c9a9e785fcf317a05c5f292d888589ee13bcdc71ffbd9d3f02","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[1,0,0,0,1,0,1],"trend_range":"2026-07-16..2026-07-22 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-16T09:01:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-dpbrz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbe3e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-zszzt.apps.aro.x0o8h6x6l7k9u1f.e0w5.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-dpbrz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbe3e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-zszzt.apps.aro.x0o8h6x6l7k9u1f.e0w5.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2080065545201258496","occurred_at":"2026-07-22T23:00:46Z","signature_id":"4a18545286c04da4906905b26b99ad474db07bd58ff38b1f675e1e3535e445a7","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}