{"meta":{"start_date":"2026-08-17","end_date":"2026-08-17","start_at":"2026-08-17T00:00:00Z","end_at":"2026-08-18T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T23:10:45Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":128,"matched_occurrences":127,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"46e30b272cd4ab3ae13194cd0f744e27ec0c9420bc1323c7d2de64c29ecb7c49","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":91,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":91,"after_last_push_count":376,"trend_counts":[0,0,92,93,50,50,91],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001710ce0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-698d8twwjg2r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cc23ee56-73dc-4ae5-8c34-8b564ec4ac36\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cc23ee56-73dc-4ae5-8c34-8b564ec4ac36\",\n      \"name\": \"cc23ee56-73dc-4ae5-8c34-8b564ec4ac36\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T02:28:22.666455393Z\",\n      \"endTime\": \"2026-08-17T02:47:22.778823921Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001710ce0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-698d8twwjg2r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cc23ee56-73dc-4ae5-8c34-8b564ec4ac36\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cc23ee56-73dc-4ae5-8c34-8b564ec4ac36\",\n      \"name\": \"cc23ee56-73dc-4ae5-8c34-8b564ec4ac36\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T02:28:22.666455393Z\",\n      \"endTime\": \"2026-08-17T02:47:22.778823921Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00108c760\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-m2k7n5wmk64r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae73c173-0301-49f0-aed1-e92e40e64280\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae73c173-0301-49f0-aed1-e92e40e64280\",\n      \"name\": \"ae73c173-0301-49f0-aed1-e92e40e64280\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T03:33:17.766641472Z\",\n      \"endTime\": \"2026-08-17T03:52:23.830264105Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00108c760\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-m2k7n5wmk64r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae73c173-0301-49f0-aed1-e92e40e64280\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae73c173-0301-49f0-aed1-e92e40e64280\",\n      \"name\": \"ae73c173-0301-49f0-aed1-e92e40e64280\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T03:33:17.766641472Z\",\n      \"endTime\": \"2026-08-17T03:52:23.830264105Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00170c160\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sjfvd9hrpkqz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5275668f-d165-4500-862f-2883c658230e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5275668f-d165-4500-862f-2883c658230e\",\n      \"name\": \"5275668f-d165-4500-862f-2883c658230e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T02:28:33.556125561Z\",\n      \"endTime\": \"2026-08-17T02:47:42.702814321Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00170c160\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sjfvd9hrpkqz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5275668f-d165-4500-862f-2883c658230e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5275668f-d165-4500-862f-2883c658230e\",\n      \"name\": \"5275668f-d165-4500-862f-2883c658230e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T02:28:33.556125561Z\",\n      \"endTime\": \"2026-08-17T02:47:42.702814321Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-marketplace-catalog, redhat-operators-catalog\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"0cb6a7bce3c1c8c9fd6bfa08d081825dbab92b4d1f2e8bb42edfa83c5e333537","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"174feb993b3522eac6a29cb08fd7e91e5aac66a120f622f7433809b1989b45ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"18634badd235c47cc7769b2415f9d9af08246303f9af555aaa25ab08d357e79f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"1970a72c4e917c4a31c4677164c7669e6a7ec59bf3ad0084014b6829828b00e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"19870c42c0b99d9a03e77c73a32110cd64ebdc7b2f1fb3b2787b840fcf69e242","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"1ad9627e2d504aa0ca75f4a4220c4674e394ba485cb8f1669a33ef075cffe710","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"2c8bdbbc95dc332e326d2cecbacb800d6252bfb66e84b0b6e4aefbce2cd84421","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"3023bf1fa9b0a8603e7363d50769c46827725c66ad4356856410af2635890c4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"33189c19af86ce920aaeca6f1ad0a17117c79e70fc2f0a5f7af1981c13e12ff3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"3d24a57f2c57ae6974c3a13f47102c5c2dd89821f12c464e20cdcb7fd40b0a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"45459f27b689b5b3261e6b1d6b08bfdb88ef34e04b1c37a8fdd920dd7f3341dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"4824ecd33024b40b5448b7da8111344e336c39879d9c8086479a0a75eab555bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"4a1b99f6300d259021d16e25d64b63e90fcd90694e4fd69c03dc115d30024b57","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"4a758add3e7296a1586eaa649fcb9c7c891946ece2583bc18cb3698ce9035fd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"4af41cfb01528200af3af738526a483f75fe69e3946b48db932c5f6e180295ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"4b066fe6b0e6afc5c0860b802e0d025953d7406f8f2373c40932592c9b4189b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"571246a094252a19e536aa70c4df678200050e4c525d65cf702200334ff8fbf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"57b3148e5307bf8b9bd2a3aa591d9b463305bab104ebfe9a67d7bfaedd819971","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"645d21739a2c607f2f097959eebcb022294146323e6c3ea5d50ad73eb10da91e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"6a83afd288f80ca5749709382b86977eed16a67172859b20c6fc0295a25d95f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"71753b948eeb762831abea32aba4ee0334f240f34ad8c4284023487572654841","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"73560923c7b148e4dc4e02d965d79d126e8b1aedc45213e58731ae3cf6eb487f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"76644a6280fa30f4b35076c06d03f97f969d25cfb86d01e5e0ea08883142c3f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"76d0aaca48d3d8b13b5a9708b151a262087ee28c82fad1c5e27162767325dea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"7aa650f91bc6e940ded19bd3c4aa084c6f4e020b3835c1d8a32fda37b5c1a31a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"81f9089b3468fcc55fda6ab32ce439c4911e90fe1bbe71fe547a03a19b9f1a5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"88a0e0feb08ba8c44934204f4cfc8528acb353c8a5d810a81cfa806598b0a09c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"89e0544d3ea3d8bb26960af6a3bd388512ad4f14bfd1243a18bdfc18bd21bd90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"8d35332c9ea228bc11bff642c482c3d0c4ad6b3b73034d348a9042825d164f3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"8e5dd8ddbf88da6a7a486b7776f1603cc11b0ea8b4693121791116d6ad16c35a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"9535d3e7fa717ed7266a74bb538d6effad7bce083b4c9ed4a6385414503c4a04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"a8515f21735c996bf85de62dad85dfb742cf7a5bf72972d08e54aee29c6d65d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"ae95583fa03c8930494be69ea80a42273ceac1f5d4206867b1a9e0d7206ec9ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"b31e24c3882a3a1b99137768e6928a68523df591535d1d71774d94e19a42eb08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"bc10a394e0c47bc331f3437732e69ef1eff8c4428177fe1b77ee52c7ac53774c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"be2bb2987dd1f10571f785bb6b740780ee91e1465f3b5d4c6022a45fc404bb08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"beff1092a16b7ba950af1abba3e1d9a87d27f3bf37701618999d4fd5cf4c2017","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"cb1f999b9c2683da7d82652800bdbb6a28756fcfe3f3472d7e9b444d948c2132","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"cb5f85dcc30bea4370d56d01cd6f539efc36342576cfa83cb21c2bc50818a193","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"cf6380def6990db7c38ca9c70a92438f52e48588f2e5eabe745c4c7f21747963","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"dc691ae7403fe5b5846b7a1fac37e6cf44fa61b56d1ae6186023d2f971f946ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"e575571df166f2f7e5b3a59cbf8a6e254fe36799c99eb6d8038b9075d6581a71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"e94947edce9d457522545e17edfde4a3b9e4e42552230a8cb1e711439a71b295","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"ed8465276418c43d64774dd8e9b713b58628ad9cf73832c732a77630d5f6e028","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"f2790d1bf47c7acba6f231ed6e5122b175b47165c05ffeb5149e42f6492c06b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"feb7de2b41b723e2687caea305b0f4a7568a4024ce4fa3b0dd9320f38f62709d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"04585aa718f71f141505064ad2fe732f705f9589d31e99639eec88515dd2a7c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"0dadb560804b7e0d901500b0f39c99252538155f3f03b03310a272ad34155f75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"0dc28d39da8ca491bc0a3747521f06704b85650170902c7ad516525881eb1af2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"123b8ad74d977192778498691cc59baf79f5b4098726550cc2544097f2eb9a82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"1771460c23a1ace2ef6fb13f6b21d26c92f4f0489015551c5c740b0dd496b6a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"36f04ac74507d6309e72813ade0e8743c6e3a31475d9907ca1d2ff182eb29731","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"4b04cc8a49d998db78514dddb6491f7ac87abb98e10d9dd5744acab6977a567b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"5010c0d66b0cdd9250af8e0ab96d2d0ce160ab1a92fed3f324f381816b592bb3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"57a4784d812f41b1e38522044cd7c96463eaa16d672e29f0785ad8eee3b3fc1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"6cca2c2f58e4be8cd0ebf41676c82fea1e76331fee7baa9bd6ad6cecb39d22e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"78d94fa03d3c9c163bdc0a4946cffc3080409afb0f4e4947d8d3b5061515c0cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"809b9a4556dee346131b6d6192b22990b80864e5c19e15397144fc49d8af3a61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"87d098d4dd922c973b8e202bf49df4861a257c3017658aec46b7c53423edd702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"884ca65c68f77e2d4a07282ffabaab81d2380e94883f4f9c72e5fe21c757f428","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"89c5e124b9e8b74c154c024e373fba30f7f651e9a49b219f2220961cbcc29246","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"89d1536fa281f6324073a4e88d63f9f26e0f0d8d7a0d90f2fbf2d596acf250e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"8d87c30e7c7db166b0ba2382009955c8dac5734d2184d8c367d553ff45e39c79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"9691eb8ba3f5db47e462e5d4eb0ae1ed9f683ccaf713e64f68aca489e29fb262","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"977f26efb6b7cde170f855d44f7d97e54d5a0cd0d99d0e75d06343598d55142c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"9ba370da71a38cec26cc27fa65f5c963d7627a2a3aed24581792ba01af05f3f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"9deb044e15702207a6ab36a04d3d0c855a3060d9949167c3a01dd3d6620b4f0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"a00c6dae0d8ba079fea8369ddfed2c1c838872b0fe3be3e2c0e2ea0f6f70e980","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"a100f768573e9ae08fa86557365071bbf813fb24aaf6dd4492308e5141fd1916","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"a1b030149d9441183312da4d4c9d1e93ea816a42448c9ed33f847d291ea26a04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"a2d73ec55c193306880a3e21652163b533da86d2920c35a29821d6051378ebae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"ab14d8aa05334e9035f211150b2681b77e6bd270e84630285e6e830442b2ccb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"ad8b70ac22e1d7d20bb6477cb653fb816f3fc7d9b46b0c9755e5264d941d61ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"ae105266a12a768699b9f003aac4bb4e3628ab8d2dd8baea8456db70179d1fdc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"b1ccf37d64e0238f61e0a0e69a74aa273f98b4ad21638c2b2afc8946f74b9927","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"b43f31ec877b5e5600bbd262dbc2a2cb367f0800ea03fccacfc0d25374fc6bee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"b6d2343644f8c24e25c49d7c01996010c6eb1ce3652f7a28be227e99931fc59e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"bf1287544bef9d5abe0f173da0d4e33524076e69e790e08094852dc919aaf822","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"c055fbe5ca6cdc75143dd736326a557299c5ae1d5b3955fdbfec3d7352180a8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"d07a7a1a9886bed1e86d24d8a4ff019408dad83e342fc1ef0771585ffcf1b79e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"d7fbcdfba2ffca0c5d589fd5ebd7dc93149ef95316d80db54060026b6d52ac21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"dc7ae60e4ab3c69f61a77cd6a4d57326c690a2d7400e12db294546d6aba40dbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"e0943d56b78ed2ab3cd68b3053d15cc102a4ec267cce30179d735082d29fc023","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"e50e1a04d0e2161c3efba204e5e41cd99ddbe7eefa1cb69476b41af7d8011588","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"e8496a54c5688e4ff20da9919a2ac87faafc38c5e0df50bca721c52437e76ea8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"e9626d70f4c04864dec32c9faaff2fd34400a839813135d371f9b0cfc47c3f87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"f0f79d27b2c13ab4c567b6014ec5f4cd8f7de38d7ad554c447b3156bff30a2c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"f12d440c00e90c38d7b8f3c07e5101718d909b224623793ee348e27327b51e98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"f404c091f5d4cf9e6f359fff8923031d3d225cc38b5e1a87434de1aaa804f892","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"fc4b2f7df507ed24abc7ccd8e33f6571d3cfd38d21be06685221d3c8daf595df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"fe0b11bc2058dc60d050271fdeee105b790414ea6c76dc8329f531a7e3c408b8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":12,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":12,"after_last_push_count":90,"trend_counts":[21,57,0,0,0,0,12],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-12T23:00:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f37000\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7fn8jpwhp4km\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d75007c4-0533-480b-9984-50b679c9adec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d75007c4-0533-480b-9984-50b679c9adec\",\n      \"name\": \"d75007c4-0533-480b-9984-50b679c9adec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:34:14.15130328Z\",\n      \"endTime\": \"2026-08-17T23:53:14.808305799Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f37000\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7fn8jpwhp4km\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d75007c4-0533-480b-9984-50b679c9adec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d75007c4-0533-480b-9984-50b679c9adec\",\n      \"name\": \"d75007c4-0533-480b-9984-50b679c9adec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:34:14.15130328Z\",\n      \"endTime\": \"2026-08-17T23:53:14.808305799Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b2f40\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-z592j4xwpxlf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62641195-f7a4-4bee-a92a-05c5558b2c33\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62641195-f7a4-4bee-a92a-05c5558b2c33\",\n      \"name\": \"62641195-f7a4-4bee-a92a-05c5558b2c33\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:43:41.693529092Z\",\n      \"endTime\": \"2026-08-18T00:02:42.709156128Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b2f40\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-z592j4xwpxlf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62641195-f7a4-4bee-a92a-05c5558b2c33\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62641195-f7a4-4bee-a92a-05c5558b2c33\",\n      \"name\": \"62641195-f7a4-4bee-a92a-05c5558b2c33\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:43:41.693529092Z\",\n      \"endTime\": \"2026-08-18T00:02:42.709156128Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-88mqt7 at version 4.22.0-0.nightly-multi-2026-08-17-010411\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0cac0\u003e: \n    failed to create HCP cluster np-skip-minor-88mqt7: failed waiting for cluster=\"np-skip-minor-88mqt7\" in resourcegroup=\"rg-np-skip-minor-88mqt7-rzrbvmhkgxzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/afc294b2-aa75-4f61-8f95-5103d3b95705\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/afc294b2-aa75-4f61-8f95-5103d3b95705\",\n      \"name\": \"afc294b2-aa75-4f61-8f95-5103d3b95705\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:39:45.286735401Z\",\n      \"endTime\": \"2026-08-17T23:58:52.406085858Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-88mqt7 at version 4.22.0-0.nightly-multi-2026-08-17-010411\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0cac0\u003e: \n    failed to create HCP cluster np-skip-minor-88mqt7: failed waiting for cluster=\"np-skip-minor-88mqt7\" in resourcegroup=\"rg-np-skip-minor-88mqt7-rzrbvmhkgxzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/afc294b2-aa75-4f61-8f95-5103d3b95705\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/afc294b2-aa75-4f61-8f95-5103d3b95705\",\n      \"name\": \"afc294b2-aa75-4f61-8f95-5103d3b95705\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:39:45.286735401Z\",\n      \"endTime\": \"2026-08-17T23:58:52.406085858Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"1cbb92b3ec49a9d46efc8c1100643ddb4ab2a3a6e8ca9362a7ab30c32a2e35b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"2fea4a039e17fda636e977a358fdae0fd8d5431cb6a1dc5f7059b3893c8c8109","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"32ffbf88f32b24bed1647b6119bb71901e54c08bd45572cf09a19833b35afb33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"472048b20a3b790bdb3415c702a99f7e65167f1bedcea097d15e70dffe32d435","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"4cd14b5b24d0d123398dbf0339cfa3e25d128a2e3f6520522d202983361aa16e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"97a0bd8ea724fff66855578fde1f4040116bcb14bc621e31109b1203d2e6861c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"a45bf7f15d4d715bd8241b053c14949c7a0fbbf894382dbe7fb53133374d3470","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"a7c56f24e4d86223f4eabaf0eb738353b2888fc5ff2c6e46e8e1f403e867dbfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"b9f4c62131caf6e8c0448727f369e4905b478b29ca061acedc9ab62c87b232fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"bd009657bc5539498ff97ef85a10834989363a2ef698cfb04d12d6f02232b73d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"c43120b306c925138893dda786102ebf997fa8e08cce37036d50ac774b7417e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"ce020347d00d259f074e4859ed6a91b27e8e3043ae287d51d85b8b31f1967757","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":4,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-04T17:43:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:160]: failed to create node pool \"np-9vv6s5\" for cluster \"cluster-nightly-4-23-9vv6s5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062ed00\u003e: \n    failed to create NodePool np-9vv6s5: failed waiting for nodepool=\"np-9vv6s5\" for cluster \"cluster-nightly-4-23-9vv6s5\" in resourcegroup=\"rg-nightly-4-23-7cwfldmpbdnh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d33a7c3b-af2e-46df-863c-7b84810d4c19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d33a7c3b-af2e-46df-863c-7b84810d4c19\",\n      \"name\": \"d33a7c3b-af2e-46df-863c-7b84810d4c19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:36:34.533538698Z\",\n      \"endTime\": \"2026-08-17T23:55:34.957276854Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:160]: failed to create node pool \"np-9vv6s5\" for cluster \"cluster-nightly-4-23-9vv6s5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062ed00\u003e: \n    failed to create NodePool np-9vv6s5: failed waiting for nodepool=\"np-9vv6s5\" for cluster \"cluster-nightly-4-23-9vv6s5\" in resourcegroup=\"rg-nightly-4-23-7cwfldmpbdnh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d33a7c3b-af2e-46df-863c-7b84810d4c19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d33a7c3b-af2e-46df-863c-7b84810d4c19\",\n      \"name\": \"d33a7c3b-af2e-46df-863c-7b84810d4c19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:36:34.533538698Z\",\n      \"endTime\": \"2026-08-17T23:55:34.957276854Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:160]: failed to create node pool \"np-kqc7kq\" for cluster \"cluster-nightly-4-21-kqc7kq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e77ba0\u003e: \n    failed to create NodePool np-kqc7kq: failed waiting for nodepool=\"np-kqc7kq\" for cluster \"cluster-nightly-4-21-kqc7kq\" in resourcegroup=\"rg-nightly-4-21-82ds5tpl6srh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c1f3559-af12-4526-953f-f7efab2099c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c1f3559-af12-4526-953f-f7efab2099c8\",\n      \"name\": \"3c1f3559-af12-4526-953f-f7efab2099c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:37:22.569020042Z\",\n      \"endTime\": \"2026-08-17T23:56:24.71939702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:160]: failed to create node pool \"np-kqc7kq\" for cluster \"cluster-nightly-4-21-kqc7kq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e77ba0\u003e: \n    failed to create NodePool np-kqc7kq: failed waiting for nodepool=\"np-kqc7kq\" for cluster \"cluster-nightly-4-21-kqc7kq\" in resourcegroup=\"rg-nightly-4-21-82ds5tpl6srh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c1f3559-af12-4526-953f-f7efab2099c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c1f3559-af12-4526-953f-f7efab2099c8\",\n      \"name\": \"3c1f3559-af12-4526-953f-f7efab2099c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:37:22.569020042Z\",\n      \"endTime\": \"2026-08-17T23:56:24.71939702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:160]: failed to create node pool \"np-vqdplg\" for cluster \"cluster-nightly-5-0-vqdplg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb4e20\u003e: \n    failed to create NodePool np-vqdplg: failed waiting for nodepool=\"np-vqdplg\" for cluster \"cluster-nightly-5-0-vqdplg\" in resourcegroup=\"rg-nightly-5-0-gdjjtfqck5vs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c5b38fd-853e-4df4-acad-76ec36f1ffeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c5b38fd-853e-4df4-acad-76ec36f1ffeb\",\n      \"name\": \"4c5b38fd-853e-4df4-acad-76ec36f1ffeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:38:36.758595452Z\",\n      \"endTime\": \"2026-08-17T23:57:45.025483153Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:160]: failed to create node pool \"np-vqdplg\" for cluster \"cluster-nightly-5-0-vqdplg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb4e20\u003e: \n    failed to create NodePool np-vqdplg: failed waiting for nodepool=\"np-vqdplg\" for cluster \"cluster-nightly-5-0-vqdplg\" in resourcegroup=\"rg-nightly-5-0-gdjjtfqck5vs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c5b38fd-853e-4df4-acad-76ec36f1ffeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c5b38fd-853e-4df4-acad-76ec36f1ffeb\",\n      \"name\": \"4c5b38fd-853e-4df4-acad-76ec36f1ffeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:38:36.758595452Z\",\n      \"endTime\": \"2026-08-17T23:57:45.025483153Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"16e65d6e25feff9df7f1c1ea49d2a18be95199cde0e35e5dc82acc567cd4b37c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"386800f08f9fe20ea792d4f9e486ef0023446318d2adb9865a8c94f18db9cf0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"72b3971a9a32041448c91ffb05bd1b7024a495dd7e1a19607c19985b70075bb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"7857c5c57296a454eb318cefdc68a4749011fd54991951874b96df86ce5b769f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"44e2d333ce62abafbaf16d046b61703140a463730eb39f1d9ff95ea9875c0757","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":15,"trend_counts":[0,2,3,3,2,2,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-10"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":15,"trend_counts":[0,2,3,3,2,2,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-10"],"prior_runs_affected":16,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"d06b61b2a290f31b2b44796608e3a50ed66dd83e5cff65620f4a4dbd778567f6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a4c7965c412b351e80ef0544bff250a8a694f957336d67b40d995be9b67f72d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-d6dm5m\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014be5a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-d6dm5m: failed waiting for cluster=\"cluster-zstream-4-21-d6dm5m\" in resourcegroup=\"rg-zstream-upgrade-4-21-5sm2l67gmw5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/95226c2f-ac1f-45b9-a636-df85c24e666e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/95226c2f-ac1f-45b9-a636-df85c24e666e\",\n      \"name\": \"95226c2f-ac1f-45b9-a636-df85c24e666e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:26.187243048Z\",\n      \"endTime\": \"2026-08-17T23:50:33.942616598Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-version-operator, cluster-storage-operator, cluster-network-operator, dns-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-d6dm5m\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014be5a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-d6dm5m: failed waiting for cluster=\"cluster-zstream-4-21-d6dm5m\" in resourcegroup=\"rg-zstream-upgrade-4-21-5sm2l67gmw5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/95226c2f-ac1f-45b9-a636-df85c24e666e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/95226c2f-ac1f-45b9-a636-df85c24e666e\",\n      \"name\": \"95226c2f-ac1f-45b9-a636-df85c24e666e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:26.187243048Z\",\n      \"endTime\": \"2026-08-17T23:50:33.942616598Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-version-operator, cluster-storage-operator, cluster-network-operator, dns-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-8ccbc2\" with version 4.22.8 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0003174c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-8ccbc2: failed waiting for cluster=\"cluster-zstream-4-22-8ccbc2\" in resourcegroup=\"rg-zstream-upgrade-4-22-t9xq7cgsp4s8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45a61bd-c6f4-4d71-a1bb-ed72a0ba3beb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45a61bd-c6f4-4d71-a1bb-ed72a0ba3beb\",\n      \"name\": \"c45a61bd-c6f4-4d71-a1bb-ed72a0ba3beb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:24.445471869Z\",\n      \"endTime\": \"2026-08-17T23:50:24.030978687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, certified-operators-catalog, cluster-policy-controller, packageserver, cluster-storage-operator, cluster-image-registry-operator, olm-operator, dns-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, catalog-operator, cluster-version-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-8ccbc2\" with version 4.22.8 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0003174c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-8ccbc2: failed waiting for cluster=\"cluster-zstream-4-22-8ccbc2\" in resourcegroup=\"rg-zstream-upgrade-4-22-t9xq7cgsp4s8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45a61bd-c6f4-4d71-a1bb-ed72a0ba3beb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45a61bd-c6f4-4d71-a1bb-ed72a0ba3beb\",\n      \"name\": \"c45a61bd-c6f4-4d71-a1bb-ed72a0ba3beb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:24.445471869Z\",\n      \"endTime\": \"2026-08-17T23:50:24.030978687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, certified-operators-catalog, cluster-policy-controller, packageserver, cluster-storage-operator, cluster-image-registry-operator, olm-operator, dns-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, catalog-operator, cluster-version-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 2 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"0d60be661dfb804031fac3a0d698877ea8db75f53d9ec7ec65ea495c39f3423b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"c9620a04091483fce1cc7d0cf73f6f7a7d60342b7147199b85311735136096a2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":143,"trend_counts":[0,0,34,34,34,34,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b96c80\u003e: \n    failed to create HCP cluster ea-listk5vbng: failed waiting for cluster=\"ea-listk5vbng\" in resourcegroup=\"ea-list-mzkqzkbmzx86\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/428d5558-84cb-42b2-9b01-d7c4a59be914\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/428d5558-84cb-42b2-9b01-d7c4a59be914\",\n      \"name\": \"428d5558-84cb-42b2-9b01-d7c4a59be914\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:37.27430246Z\",\n      \"endTime\": \"2026-08-17T23:50:44.242662184Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b96c80\u003e: \n    failed to create HCP cluster ea-listk5vbng: failed waiting for cluster=\"ea-listk5vbng\" in resourcegroup=\"ea-list-mzkqzkbmzx86\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/428d5558-84cb-42b2-9b01-d7c4a59be914\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/428d5558-84cb-42b2-9b01-d7c4a59be914\",\n      \"name\": \"428d5558-84cb-42b2-9b01-d7c4a59be914\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:37.27430246Z\",\n      \"endTime\": \"2026-08-17T23:50:44.242662184Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ef700\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-w54dxhdxrw47\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe214750-2f23-4eec-acde-0e7cc499686c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe214750-2f23-4eec-acde-0e7cc499686c\",\n      \"name\": \"fe214750-2f23-4eec-acde-0e7cc499686c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:32.312465755Z\",\n      \"endTime\": \"2026-08-17T23:50:33.738372291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ef700\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-w54dxhdxrw47\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe214750-2f23-4eec-acde-0e7cc499686c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe214750-2f23-4eec-acde-0e7cc499686c\",\n      \"name\": \"fe214750-2f23-4eec-acde-0e7cc499686c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:32.312465755Z\",\n      \"endTime\": \"2026-08-17T23:50:33.738372291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"21a913684aec93b1acfbcb982e8dc4b1b9a4995b24941af4316c7c9a412465a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"584c55bbc5729e596f61f16b7ba3f43aafd93f7484b3a4a4202a0d3036ac902c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"12526f218c28804ab8ce44fb5444a8483527f506dc5e5d55e2de2d40b1678b14","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0026382e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-rz9kcmxlzjrl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/885c8055-6ed5-453a-9848-a3fc31887be1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/885c8055-6ed5-453a-9848-a3fc31887be1\",\n      \"name\": \"885c8055-6ed5-453a-9848-a3fc31887be1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:35:24.304672146Z\",\n      \"endTime\": \"2026-08-17T23:59:32.599329473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s8kbhn4icvjoqhhk0hha9movcd177h4 still exists (deletion dispatched at 2026-08-17T23:35:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0026382e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-rz9kcmxlzjrl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/885c8055-6ed5-453a-9848-a3fc31887be1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/885c8055-6ed5-453a-9848-a3fc31887be1\",\n      \"name\": \"885c8055-6ed5-453a-9848-a3fc31887be1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:35:24.304672146Z\",\n      \"endTime\": \"2026-08-17T23:59:32.599329473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s8kbhn4icvjoqhhk0hha9movcd177h4 still exists (deletion dispatched at 2026-08-17T23:35:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010f8390\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-4wtqz6dr4tdf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd2dc60-025b-4b60-9476-c114aff753c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd2dc60-025b-4b60-9476-c114aff753c6\",\n      \"name\": \"ffd2dc60-025b-4b60-9476-c114aff753c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:27:41.387689474Z\",\n      \"endTime\": \"2026-08-17T23:51:44.697499563Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s8kagsoquavg85b3rmljqn17gs8ol5m still exists (deletion dispatched at 2026-08-17T23:27:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010f8390\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-4wtqz6dr4tdf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd2dc60-025b-4b60-9476-c114aff753c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd2dc60-025b-4b60-9476-c114aff753c6\",\n      \"name\": \"ffd2dc60-025b-4b60-9476-c114aff753c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:27:41.387689474Z\",\n      \"endTime\": \"2026-08-17T23:51:44.697499563Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s8kagsoquavg85b3rmljqn17gs8ol5m still exists (deletion dispatched at 2026-08-17T23:27:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"cfbc010ad0f4e1047ad0ea6c2d89eeacd1510923705b56e38f306d8e9240411c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"fa03ca8bf9eea31e53d8ba0acda7b875b5bf8ccbcd76ee0131b50f654dc45123","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"da65f908f97313ddbcc1cd3d11e952371e136eb64bf37368bcd0011d3f5a924e","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": context deadline exceeded","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.106.185:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.106.185:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,1,1,1,0,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-15T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-pxssnz\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fe1e78\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.106.185:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.106.185:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-pxssnz\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fe1e78\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.106.185:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.106.185:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc00157eb40\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.231.127:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.231.127:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc00157eb40\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.231.127:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.231.127:443/apis/packages.operators.coreos.com/v1\": context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"1c3cab581e31ca308638d0dbe2389111141fab2980bdce8aa655fa5c2e5056a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089170383700758528","occurred_at":"2026-08-17T02:00:08Z","signature_id":"b2f0a68375b2ac63fdbe9a8de2bedad766e492b3141a3e7cf65741c280078cae","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-cgkj2b, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-cgkj2b in resource group rg-nightly-5-1-xwxjbkpqbndz, error: failed waiting for cluster=\"cluster-nightly-5-1-cgkj2b\" in resourcegroup=\"rg-nightly-5-1-xwxjbkpqbndz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-cgkj2b in resource group rg-nightly-5-1-xwxjbkpqbndz, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-08-03","2026-08-10"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-xwxjbkpqbndz/cluster-nightly-5-1-cgkj2b should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001808300\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-cgkj2b, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-cgkj2b in resource group rg-nightly-5-1-xwxjbkpqbndz, error: failed waiting for cluster=\"cluster-nightly-5-1-cgkj2b\" in resourcegroup=\"rg-nightly-5-1-xwxjbkpqbndz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-cgkj2b in resource group rg-nightly-5-1-xwxjbkpqbndz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-xwxjbkpqbndz/cluster-nightly-5-1-cgkj2b should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001808300\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-cgkj2b, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-cgkj2b in resource group rg-nightly-5-1-xwxjbkpqbndz, error: failed waiting for cluster=\"cluster-nightly-5-1-cgkj2b\" in resourcegroup=\"rg-nightly-5-1-xwxjbkpqbndz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-cgkj2b in resource group rg-nightly-5-1-xwxjbkpqbndz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a98ba0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fznbzbhmrll6, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fznbzbhmrll6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fznbzbhmrll6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a98ba0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fznbzbhmrll6, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fznbzbhmrll6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fznbzbhmrll6, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"2e918afb8bab4d60bf9c98d80c6e2dc2ffc48453cc51b7f3e8ce8fad15338dfd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"cf04cfea39170c77aed93ac0ff611c2f7ed2e9b7f99039c1d5712d2e36737d03","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-dgzvb2mbxd5c, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-11T23:00:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-dgzvb2mbxd5c\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002e4a210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-dgzvb2mbxd5c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-dgzvb2mbxd5c\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002e4a210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-dgzvb2mbxd5c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"36b8044b32cef4a949255906c94fe8bbc27aba77678125beaabc35a37f07e459","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"dd07db87a89697d9c67bf7c6683c117b7452caa1c79d38efee027f1f10aa8459","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-7nmln6 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f10cc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7nmln6: failed waiting for cluster=\"np-version-upgrade-cluster-7nmln6\" in resourcegroup=\"rg-np-version-upgrade-7nmln6-k86zrxnsfjs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2ee6fba-ea75-4564-aae2-265d7e31576b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2ee6fba-ea75-4564-aae2-265d7e31576b\",\n      \"name\": \"b2ee6fba-ea75-4564-aae2-265d7e31576b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T01:25:22.703111014Z\",\n      \"endTime\": \"2026-08-17T01:44:32.09498305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-7nmln6 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f10cc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7nmln6: failed waiting for cluster=\"np-version-upgrade-cluster-7nmln6\" in resourcegroup=\"rg-np-version-upgrade-7nmln6-k86zrxnsfjs6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2ee6fba-ea75-4564-aae2-265d7e31576b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2ee6fba-ea75-4564-aae2-265d7e31576b\",\n      \"name\": \"b2ee6fba-ea75-4564-aae2-265d7e31576b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T01:25:22.703111014Z\",\n      \"endTime\": \"2026-08-17T01:44:32.09498305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"4dc2b993957feba0a3cd05482c5f3923fe6d3aac805903745a04ff6d07ec961a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"57f7bb5542684c84c14b4b2501a5b749f4f6ed6cf03fb8784c04f360a07fbb7f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-db9z9w\" with version 4.20.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7b220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-db9z9w: failed waiting for cluster=\"cluster-zstream-4-20-db9z9w\" in resourcegroup=\"rg-zstream-upgrade-4-20-m7nb92j4xs2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9225139d-78e2-489a-939f-7ea524a5b00d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9225139d-78e2-489a-939f-7ea524a5b00d\",\n      \"name\": \"9225139d-78e2-489a-939f-7ea524a5b00d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:27.308208641Z\",\n      \"endTime\": \"2026-08-17T23:50:33.971903028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-db9z9w\" with version 4.20.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7b220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-db9z9w: failed waiting for cluster=\"cluster-zstream-4-20-db9z9w\" in resourcegroup=\"rg-zstream-upgrade-4-20-m7nb92j4xs2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9225139d-78e2-489a-939f-7ea524a5b00d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9225139d-78e2-489a-939f-7ea524a5b00d\",\n      \"name\": \"9225139d-78e2-489a-939f-7ea524a5b00d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:31:27.308208641Z\",\n      \"endTime\": \"2026-08-17T23:50:33.971903028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"5601d847b3ca77de025dc15cc14baa9957a11fcbdf1cce524903b80a77473545","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[3,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-11T23:00:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001314348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-clv99v\" in resourcegroup=\"rg-zstream-upgrade-4-19-j449bl8tj972\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcf567de-94ae-48cb-8eb1-ef6e1563ba32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcf567de-94ae-48cb-8eb1-ef6e1563ba32\",\n      \"name\": \"bcf567de-94ae-48cb-8eb1-ef6e1563ba32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:39:29.020988992Z\",\n      \"endTime\": \"2026-08-18T00:03:32.52458889Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s8kn4ufdhtfgtp1a2s0bfr847bntrhh still exists (deletion dispatched at 2026-08-17T23:39:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001314348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-clv99v\" in resourcegroup=\"rg-zstream-upgrade-4-19-j449bl8tj972\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcf567de-94ae-48cb-8eb1-ef6e1563ba32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcf567de-94ae-48cb-8eb1-ef6e1563ba32\",\n      \"name\": \"bcf567de-94ae-48cb-8eb1-ef6e1563ba32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-17T23:39:29.020988992Z\",\n      \"endTime\": \"2026-08-18T00:03:32.52458889Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s8kn4ufdhtfgtp1a2s0bfr847bntrhh still exists (deletion dispatched at 2026-08-17T23:39:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089487626758262784","occurred_at":"2026-08-17T23:00:45Z","signature_id":"a8d5cadabd2f5928ac91b3e7242f10bfd9320d8ca703d3812f2198351a3064e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab9e82a8af804bbb7c138751caa26c2cb132cf89c97d2f090acc75dc9a8b2e15","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": http: server gave HTTP response to HTTPS client","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.35.189:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.35.189:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-16T02:00:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e8e7e0\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.35.189:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.35.189:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:163]: failed to verify HCP cluster viability for back-level version 4.19\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e8e7e0\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.35.189:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.35.189:443/apis/packages.operators.coreos.com/v1\": http: server gave HTTP response to HTTPS client\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"26fabdf2582e99854fdaa16ed8fc0c4b69e915c990ab6cbd981df08d2b373866","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6477f6d87ddc79ca469b892f0755a20abbcbf9857b736992068b4e3c57909813","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from \u003curl\u003e Get \"\u003curl\u003e\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.125.72:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.125.72:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-rh8468\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000dc9bd8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.125.72:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.125.72:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: failed to verify HCP cluster \"cluster-zstream-4-19-rh8468\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000dc9bd8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1.packages.operators.coreos.com is not available because: failing or missing response from https://10.130.125.72:443/apis/packages.operators.coreos.com/v1: Get \"https://10.130.125.72:443/apis/packages.operators.coreos.com/v1\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089145955646443520","occurred_at":"2026-08-17T00:23:04Z","signature_id":"3045c9b8afef48ebd4bbcfeaa7399ab790157ab947236fa08ec61687b95bddeb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}