{"meta":{"start_date":"2026-08-16","end_date":"2026-08-16","start_at":"2026-08-16T00:00:00Z","end_at":"2026-08-17T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T00:12:01Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":95,"matched_occurrences":95,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"46e30b272cd4ab3ae13194cd0f744e27ec0c9420bc1323c7d2de64c29ecb7c49","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":50,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":50,"after_last_push_count":285,"trend_counts":[0,0,0,92,93,50,50],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-sxbphn\" with version 4.20.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009f0260\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-sxbphn: failed waiting for cluster=\"cluster-zstream-4-20-sxbphn\" in resourcegroup=\"rg-zstream-upgrade-4-20-pfzcshdwfdz6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5941e8cc-cc16-48f8-b543-97eb9dae1042\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5941e8cc-cc16-48f8-b543-97eb9dae1042\",\n      \"name\": \"5941e8cc-cc16-48f8-b543-97eb9dae1042\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:34:01.762150251Z\",\n      \"endTime\": \"2026-08-16T23:53:03.389351771Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-sxbphn\" with version 4.20.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009f0260\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-sxbphn: failed waiting for cluster=\"cluster-zstream-4-20-sxbphn\" in resourcegroup=\"rg-zstream-upgrade-4-20-pfzcshdwfdz6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5941e8cc-cc16-48f8-b543-97eb9dae1042\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5941e8cc-cc16-48f8-b543-97eb9dae1042\",\n      \"name\": \"5941e8cc-cc16-48f8-b543-97eb9dae1042\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:34:01.762150251Z\",\n      \"endTime\": \"2026-08-16T23:53:03.389351771Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-0-4m98wphsz5rk/cluster-nightly-5-0-srrl9h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef8860\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-srrl9h: failed waiting for cluster=\"cluster-nightly-5-0-srrl9h\" in resourcegroup=\"rg-nightly-5-0-4m98wphsz5rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/08b22b3c-e2be-4b41-8e2d-c8e5b126a57d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/08b22b3c-e2be-4b41-8e2d-c8e5b126a57d\",\n      \"name\": \"08b22b3c-e2be-4b41-8e2d-c8e5b126a57d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:33:39.83194699Z\",\n      \"endTime\": \"2026-08-16T23:52:43.352394655Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog, community-operators-catalog, redhat-marketplace-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-0-4m98wphsz5rk/cluster-nightly-5-0-srrl9h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef8860\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-srrl9h: failed waiting for cluster=\"cluster-nightly-5-0-srrl9h\" in resourcegroup=\"rg-nightly-5-0-4m98wphsz5rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/08b22b3c-e2be-4b41-8e2d-c8e5b126a57d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/08b22b3c-e2be-4b41-8e2d-c8e5b126a57d\",\n      \"name\": \"08b22b3c-e2be-4b41-8e2d-c8e5b126a57d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:33:39.83194699Z\",\n      \"endTime\": \"2026-08-16T23:52:43.352394655Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog, community-operators-catalog, redhat-marketplace-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-trbcdj\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e92620\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-trbcdj: failed waiting for cluster=\"cluster-zstream-4-21-trbcdj\" in resourcegroup=\"rg-zstream-upgrade-4-21-wqnvsmvszc87\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ff63878-7688-4ff7-86a0-57af5814a9bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ff63878-7688-4ff7-86a0-57af5814a9bf\",\n      \"name\": \"3ff63878-7688-4ff7-86a0-57af5814a9bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:34:01.856863881Z\",\n      \"endTime\": \"2026-08-16T23:53:03.349730024Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-trbcdj\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e92620\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-trbcdj: failed waiting for cluster=\"cluster-zstream-4-21-trbcdj\" in resourcegroup=\"rg-zstream-upgrade-4-21-wqnvsmvszc87\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ff63878-7688-4ff7-86a0-57af5814a9bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3ff63878-7688-4ff7-86a0-57af5814a9bf\",\n      \"name\": \"3ff63878-7688-4ff7-86a0-57af5814a9bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:34:01.856863881Z\",\n      \"endTime\": \"2026-08-16T23:53:03.349730024Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"449ce03cc29b57777939af170bd0642d573a6f2182c759e8fda6ab64d1e9c507","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"4a90ac8b3d49ff874dd3ba17b5e92034d0b57f84aaccfa2f8a68b049f361fffe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"531669d933d9e04125172fadc2c7a98d86f4c0e7f7edebc80a77409e0c47f6c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"96976d38fb3e50beb8fc3ff0a8cf926bc85d00fac514d8c754a23e5a6c665157","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"02c9012641f92014b9c73f3fc2afa003c882a69fb6d74d37f99def204fc65106","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"034fefcb161dc899bff0c530ae91b1a232816231c9edd709a3df7a7722791e1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"04ca4994cb9f6e58db252305b0fc73da42c736f208ef1b8575ab4890d55f8f7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"08c2621b4e07e4d068e6f46529b82bba7d2134f038896cf0048f5192455cb57a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"16da7e936b9feab5d07f4db2496ad7f24dcc1825ddc432beb0180efa94374874","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"1c67a39f2a5b2e765207b8cff36642046fdc0de1c9b39071755c785b9447735f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"27c038f0ab169a753358de4f5262e2e333f8551b993acb6d7bb53add700a5a9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"31a13ff145182f3646bb751c4d4a19685e4f1d677e99c895ffa6985fd951c679","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"34af0be64b94dff110f5c115634b4cd52a8ae0fa26c41bb3161724a5ffd505bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"36b997c1e4b1e790432947971b8266f0938a38e2a9de771f57927de7d3f6edd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"37e64b4c4b43de63378e7256876cd9b4ce991e78f14f231338fcf524078a0b43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"394097d5a295d74a881545b936a6af17d276106c80ce86b93d26b617c252ee08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"3bd0a40ce40ac27eacc164c95d4c5a8558e07a1e647903f92af116079b739e9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"412ddbf356622ee6dc927e0a54831323bc2223818877e1a14d7767003a76cc61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"424424b28b3938a51a0544d47ad592065a39900580c4b5adbc53d2aa7ef5bdd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"44eb3249fe793ca4f1b1f691bb40afa3a39b568343e8cfa3a79fc9c443e57d1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"5051e188a1671b80baa973052f30b8b291f2ec0d26aa1518048b1ed72599991b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"55ba316650af58914bd0ad6bb0e13e5c3089fad1f3b6ee1d1d77c3b3dcd94e2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"5debd691a0f7d5fbcf870985d3019bab71208883cd422763db02683d36e3d1ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"60a97a34c6b1f26e3f0329e453ca288362c7fad9d2df04a8c1f223e140bbd118","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"6e076974be729ad687806edb0ebc1afa2683b6b87d5ad87f215e320423ccff57","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"6fde101d5db185e9c4c342e65447c2580d7aff1dae7e435c20272f2b330a0512","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"7364a1b694d5d1205aa80d450025ea0f26147a6b2082ac40013992253d553547","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"75f12dbf9b3b6d5209d03f04f0e6ed5474052368f023bf547b12d78e4b0be03d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"76e6e9d87effcb22af719702fc6810adbf1273f98fd32665b0c2591aecaefb6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"819d469e2a09210a593f9313409ce9006a5f3a5ccf4b05ebee9f4203c0535259","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"86733dfde8ad5dd64b827e46b9c962626103b1dc587dc0794a227470320a560e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"8a1345c81c90a0f6283791926c0693bd8677371b50d41b5766fa54fded46b885","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"9245362087ed06614868021da2ac758aada33572538cab0f71acd4bab7258110","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"97ad74b6247419988dba958b629d8d3e32e38ad4ddb1728a3ffaff15bddf1444","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"98a9094840d62eda648b45e46b9a9515a7480667302ca0f223e876130f6d0f98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"adf38936fadcafe9bae6a9e9ddf1d1d422ade8c2e27782a4f772fb7b3e2657a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"aea054804b1ea4b730fc00cbf0ac1647454ad66a79a3a387458abb423fea7597","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"af6d805396246d7ae7f22f38680f9966bee90225ff866e1205a74eb9b0f81777","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"afe8d24901ad7a2e0aba6826dade41cfdb33914d7b99e1ec3b7484e600ee58ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"b7aa28b0728982bcc0585d1d62ed86fb9dd6e46d796b0a15649cb5ff811a3bc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"c230f410c640fb8c3ccefc98eaedb03f5cb42821eb736d4daa17996b6318614c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"c24e3771a564ebcb791473f3aa1a4da7bddf9201a9cff5a3b804d96a13978bab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"ca88e2595128625e68bc4caaedfb68eb1e1e15c82135fefaed28d8abf8bbde9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"cb485be486ade64e39896243dbd32267a7745404b8c428fce698cfb0a263a1da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"d90f49db1e4229bdb2bc890276fee8d47995fb95de182ae9dc8ad74daa171bcc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"dcc9c5c715c2dea9993a6e7e22b5da8d55f6331df33388aa2d61dc258294dc0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"e5a61af4a6d6440bbdd3b2835b98892554a1a090904d7e6b93bad79cdf90e9ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"f1f43a94692ae9c324268ad067cf8d30fc513b648fb176c730ea87daec3d7bbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"f61beec391c45dfb7be458adc981996c0a936865f569d8acaa3f687191d1c8e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"fbae7acd5386b411d58f4021402bc0ccd0f16f4dad1065bf4de7ecd98261f1a4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":34,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":34,"after_last_push_count":141,"trend_counts":[0,0,0,34,34,34,34],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-03"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-07T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011d0700\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ld8fskvcf2g2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/60d623fe-efeb-4bb7-9bfc-f131505d0518\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/60d623fe-efeb-4bb7-9bfc-f131505d0518\",\n      \"name\": \"60d623fe-efeb-4bb7-9bfc-f131505d0518\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:35:10.138318354Z\",\n      \"endTime\": \"2026-08-16T23:54:13.551823532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011d0700\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ld8fskvcf2g2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/60d623fe-efeb-4bb7-9bfc-f131505d0518\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/60d623fe-efeb-4bb7-9bfc-f131505d0518\",\n      \"name\": \"60d623fe-efeb-4bb7-9bfc-f131505d0518\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:35:10.138318354Z\",\n      \"endTime\": \"2026-08-16T23:54:13.551823532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e6c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-x2w72vljtvnx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64f8e29a-b9f7-4949-9d80-17e8ed265dd7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64f8e29a-b9f7-4949-9d80-17e8ed265dd7\",\n      \"name\": \"64f8e29a-b9f7-4949-9d80-17e8ed265dd7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:03:39.564030321Z\",\n      \"endTime\": \"2026-08-16T23:22:42.66675779Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e6c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-x2w72vljtvnx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64f8e29a-b9f7-4949-9d80-17e8ed265dd7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64f8e29a-b9f7-4949-9d80-17e8ed265dd7\",\n      \"name\": \"64f8e29a-b9f7-4949-9d80-17e8ed265dd7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:03:39.564030321Z\",\n      \"endTime\": \"2026-08-16T23:22:42.66675779Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000feec60\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-jvgr88jb6956\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abe67b8f-4d20-469a-be13-28aeeddb205f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abe67b8f-4d20-469a-be13-28aeeddb205f\",\n      \"name\": \"abe67b8f-4d20-469a-be13-28aeeddb205f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:02:06.281538698Z\",\n      \"endTime\": \"2026-08-16T23:21:12.066583928Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000feec60\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-jvgr88jb6956\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abe67b8f-4d20-469a-be13-28aeeddb205f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abe67b8f-4d20-469a-be13-28aeeddb205f\",\n      \"name\": \"abe67b8f-4d20-469a-be13-28aeeddb205f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:02:06.281538698Z\",\n      \"endTime\": \"2026-08-16T23:21:12.066583928Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"003b0230839aa586a15550c4b4ff580e7b9cbdc42961eb022965a0ec548c1530","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"06f652ef9d2e2705859dabb7f1e03419f274a2157a8c643b9428545cd6536b02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"0d33e92cd179dc52ef5f26d468e913be86a6532308c67594df088b3e3a5e2071","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"15354cf25b6a8bcd711c3b1265c6beabd3113e0b3fb6f55795a8efd549cb58a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"16ba0d536122d35cc4f5e18f25dcc61f480021098a607f4bef39b4d6879a5e13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"24a5572f44702de92a1966a2b2d59c5eef4bc6d624202247f04d7afdadc3a8a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"34a896905efe8d485f7e8078b04b9524aff783b2011049b92800d5568b10ce94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"35758996aae01cccf4c6e96b2e924cf1538367ceaaecb444535d1f82d9fc33f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"36fff106072ad292e08870b430116232a8db77724cf403d6cbd8f4386c6ba12e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"3809b8d9550ee7d5310e17229cebba4d4bcd068f6bac3d793665c75b840108eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"41520944fcf0c5542de9146e5eac5f6217a259963db416ea8941f8f59487ea39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"4723088e56a2945f6feab4595236424cec975fec3f6a06d309e773742652423d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"5547b42c2d86419bc113cb0701c08aa9e2c6dafdb0b7fc4bdafee8603a18fdd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"597a310cde858a7549ece71014e06bf9dc7a716d3af6bcf62edc24d013f0a6de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"598440b335735ff1510cebf5248c8deb83596637da040bf0cb5f5ffd64f9f09d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"5efa93c7244cc794edeee22ca534104cbda203e7b26cd1fa3cad05655ef08d64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"5f2cf48e57dda932a1670edf8c49237d4b08db773e7bdbb6bc834a259e496b63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"6ea81d6cdf2e1b186b22cab793cbf9abccb94bab31184590de56d76c75e264d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"72aa8aed7aea5cddc5021827fddee9933fd5d8468fc7d006747d21f1b0b4d27d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"778d45d2c9930a4b2d8d83ed4b6a0810e227220e438114d785516bcb18d4b95c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"7b5d815d03d1e731078a61eabe916ace6c916faebc8a2f2e74380ef7c33907bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"913998413f44e6f0bde61e8fc03fe282d2ef577628462f7e12e071847ab46560","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"91952d61728c8eb47ceb32b02a2b83c5c7eb3331110b2a3167a847ad2857f218","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"9818f80a2e28ec380c32523836c96df5585b193b32c452bd239be294e7a90c54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"9a9ea1d20f7e45d7cd2e61f33003ac7bdf470829f1c65d68f5fb36310321b717","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"a16382cf7fda742610ea70d9aa0c0ab3a200aa8f83c19c57e4145d6e76eb6a31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"a2a0c929f2dffb7c42cda4914357993c51a33255985ac7b70d4e9235c2b2b96b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"b134d8e1460ace24cb98bf227aa2828ee8aaa1f55bc6d5430565dcfebc1281e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"bae4eb1e17616a07f23b19e4540e033fb1c0b652153f6c163b1c0b4764a0c610","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"c2af1a432f31ea2261d7dd77eec56d827d8f00ff754f2bd518cf03d5cb8311bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"caaa4067f6e2c4788bacd48b217de12311f6bac06ec22c72b9aab2be513961d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"caffd7dbfa4ad3c828b38d14282f67674b0d4f3f83c2ea81210b94bd6d95a531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"d30756900d4a0f2dd9d57a249b943f0da2d51a63244a347185c89c48087a4a86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"ea85e758c1ef78d1cac2c876173364dc3f0a600df6f0b2b003af07768526e7f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"44e2d333ce62abafbaf16d046b61703140a463730eb39f1d9ff95ea9875c0757","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":13,"trend_counts":[0,0,2,3,3,2,2],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":13,"prior_last_seen_at":"2026-07-29T02:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":14,"trend_counts":[0,0,2,3,3,2,2],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-07-27"],"prior_runs_affected":14,"prior_last_seen_at":"2026-08-01T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"fcfdc5b4bc3fe090403196c4f927a6fa0b757ffb90c8e6e68af29236ea54edb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab9e82a8af804bbb7c138751caa26c2cb132cf89c97d2f090acc75dc9a8b2e15","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": http: server gave HTTP response to HTTPS client","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.239.160:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.239.160:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc00137ad08\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.91.196:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.91.196:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc00137ad08\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.91.196:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.91.196:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-t4gcmd\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015f16e0\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.239.160:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.239.160:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-t4gcmd\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015f16e0\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.239.160:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.239.160:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"4cc8c1e67eab352bfb4d83eb73712b0e71375c0b65662970a8e436da244100fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2088808202962472960","occurred_at":"2026-08-16T02:00:58Z","signature_id":"7223d401dcec9e88b71ee24b428de38d112c35f705d1a0efec1f7d38b1c9da97","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6477f6d87ddc79ca469b892f0755a20abbcbf9857b736992068b4e3c57909813","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.102.137:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.102.137:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,1,2],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-w6drcq\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc001610b10\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.102.137:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.102.137:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-w6drcq\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc001610b10\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.102.137:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.102.137:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc00132bab8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.158.195:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.158.195:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc00132bab8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.158.195:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.158.195:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"1990a45efb0c0ad4da063f8c4926b8f9e3d74d174241e778dfbd9e093b988e48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"c192f2bede3d9b096e0ec6b52c5e18e9b7e9c83a6ac21b69622441a543a7d18a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-679hvq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-679hvq in resource group rg-nightly-5-1-gwjvtrzmcbks, error: failed waiting for cluster=\"cluster-nightly-5-1-679hvq\" in resourcegroup=\"rg-nightly-5-1-gwjvtrzmcbks\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-679hvq in resource group rg-nightly-5-1-gwjvtrzmcbks, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-13","2026-07-20","2026-08-03"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-09T23:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-gwjvtrzmcbks/cluster-nightly-5-1-679hvq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0006f93b0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-679hvq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-679hvq in resource group rg-nightly-5-1-gwjvtrzmcbks, error: failed waiting for cluster=\"cluster-nightly-5-1-679hvq\" in resourcegroup=\"rg-nightly-5-1-gwjvtrzmcbks\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-679hvq in resource group rg-nightly-5-1-gwjvtrzmcbks, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-gwjvtrzmcbks/cluster-nightly-5-1-679hvq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0006f93b0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-679hvq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-679hvq in resource group rg-nightly-5-1-gwjvtrzmcbks, error: failed waiting for cluster=\"cluster-nightly-5-1-679hvq\" in resourcegroup=\"rg-nightly-5-1-gwjvtrzmcbks\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-679hvq in resource group rg-nightly-5-1-gwjvtrzmcbks, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001de2270\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-hnkmsswncxq5, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hnkmsswncxq5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-hnkmsswncxq5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001de2270\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-hnkmsswncxq5, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hnkmsswncxq5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-hnkmsswncxq5, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"d6ef0371c14a10bd70bb1b67925e2bb8bcc482440b036c84d63e1828e365528a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"f217b58242c7c6b9f91397c10a9efc3b118902642385120c8b75a9c1af9d7dda","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"077f77b9825f03821964d99637a7414de7ca52612e9a64bf240378e696ca5436","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,1,1,1,1],"trend_range":"2026-08-10..2026-08-16 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-23-nfdkv46445wt/cluster-nightly-4-23-lhwwdn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f08600\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-lhwwdn: failed waiting for cluster=\"cluster-nightly-4-23-lhwwdn\" in resourcegroup=\"rg-nightly-4-23-nfdkv46445wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/18080b62-ccaa-4769-9b6c-3f353efb15be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/18080b62-ccaa-4769-9b6c-3f353efb15be\",\n      \"name\": \"18080b62-ccaa-4769-9b6c-3f353efb15be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:33:39.042449187Z\",\n      \"endTime\": \"2026-08-16T23:52:43.325318781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, certified-operators-catalog, community-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-23-nfdkv46445wt/cluster-nightly-4-23-lhwwdn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f08600\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-lhwwdn: failed waiting for cluster=\"cluster-nightly-4-23-lhwwdn\" in resourcegroup=\"rg-nightly-4-23-nfdkv46445wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/18080b62-ccaa-4769-9b6c-3f353efb15be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/18080b62-ccaa-4769-9b6c-3f353efb15be\",\n      \"name\": \"18080b62-ccaa-4769-9b6c-3f353efb15be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-16T23:33:39.042449187Z\",\n      \"endTime\": \"2026-08-16T23:52:43.325318781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, certified-operators-catalog, community-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089125084349337600","occurred_at":"2026-08-16T23:00:08Z","signature_id":"5bf13c22f909863629961c91633dc5a66e14dd3b29e252563ca5266d92dec46e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}