{"meta":{"start_date":"2026-04-09","end_date":"2026-04-09","start_at":"2026-04-09T00:00:00Z","end_at":"2026-04-10T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T18:04:26Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":5,"raw_occurrences":62,"matched_occurrences":61,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":22,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":22,"after_last_push_count":22,"trend_counts":[0,0,0,0,0,0,22],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":16,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001736fc0\u003e: \n failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qmrn9q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001736fc0\u003e: \n failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qmrn9q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b62f90\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-mvd2qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b62f90\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-mvd2qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f54d20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f54d20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"1434b8b1beae1e2462f904ed8c7fd391d5c039e029a420e6cf6d081b0c0d0b0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"1ff88c7cdd3499f0b611863b832cf641a1174675f55466b3a9b7339a91830ff1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"20d80203cbad1be4b26371494dcc6cc544922c19a12c074a01d8bb2de59736ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"2ee08db4e3b091585cb0d893f015ff959a35d3c08ed6f9c2c96d9d598749655e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"330cb64c3c554ccfae42debfd0d76fd8797f0c0fcaa880bdd959f65399919c88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"4b476ec19b339bba387ec9743367a687655cf83e55ffc3d32165311f768789cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"4bd17e5262806660ea3cb7307dacae443f38c55f69a35963fbf5b93d093d7077","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"5143f678069366b9c72b55d545d3d5ed863e67f5752c369766a1f2194e58b299","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"51d263ce91b4409e4460d92f782dcbcf48a02d5d56f79d54b7af9e35ec33c3be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"66f5e30a0278bda3bbd019ac2f4b9bb0e570da30f3b70999fa97794a1ccf19be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"763cc5ea46d3724878f97a7c2aeb317ebbf14cfa39e2f628b3b2132b6b57ae00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"9fa9e212bfc5e515204db2c8cda1c2e9df00f18528747dd58502103dd58209f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"a3601dc891786f1e347882a854cda79072206de659df846624d91488873d6788","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"ac3b4a72760a724c1b769b45f8e59a39b612336b94a6c45e7386811540975018","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"b8be7be9b69c391c1558c0d3fba34bce9063369ea28ca2c53d29ce2a33a4fdfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"ceabc22814b9276f1ff45e9ae3bafd075ad167466b20198195bf855b7b2d2c52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"d7722e9b5999c253e78f65c34ee8909e1f5f33ad52d970eaea64c9e567cf6c35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"d77696467a2d442aa450fda4be7fa109a0efae503aae9c34c88fb70b01c15bb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e02eea0acb0857847efd731148c95427f368bed721e2406ad6b6e8de41fad190","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e1c9b11aabb04b01cf379a5ddb36533a49ce1283f8171a5e595ace8671ce7942","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e777e665a1ca73ae9f55c94dd841945efb9238d4af249f7d2ae35961c1bdc2c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"f04a5eed1e2f72f7a0d6b6ed6f9ad8509a8864946852b1905eb5300b43ce6b51","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":19,"runs_affected":3,"failed_runs":3,"run_impact_percent":33.333333333333336,"anchor_occurrences":19,"after_last_push_count":68,"trend_counts":[6,0,3,1,16,0,19],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":23,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785307 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n \"ts\"=\"2026-04-10 01:41:47.102900\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-not-be-able-to-perform-various-invalid-operations-on-cluster-resources|pid:930\"\n \"ts\"=\"2026-04-10 01:41:49.288805\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"2s\"\n STEP: creating a resource group @ 04/10/26 01:41:49.288\n \"ts\"=\"2026-04-10 01:41:49.288894\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\n STEP: creating cluster parameters @ 04/10/26 01:41:54.08\n STEP: creating customer resources @ 04/10/26 01:41:54.081\n STEP: creating the HCP cluster @ 04/10/26 01:43:05.737\n \"ts\"=\"2026-04-10 01:43:05.737229\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785307 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n \"ts\"=\"2026-04-10 01:41:47.102900\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-not-be-able-to-perform-various-invalid-operations-on-cluster-resources|pid:930\"\n \"ts\"=\"2026-04-10 01:41:49.288805\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"2s\"\n STEP: creating a resource group @ 04/10/26 01:41:49.288\n \"ts\"=\"2026-04-10 01:41:49.288894\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\n STEP: creating cluster parameters @ 04/10/26 01:41:54.08\n STEP: creating customer resources @ 04/10/26 01:41:54.081\n STEP: creating the HCP cluster @ 04/10/26 01:43:05.737\n \"ts\"=\"2026-04-10 01:43:05.737229\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","ity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.976307\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.978331\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.979738\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.982005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.984519\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.986359\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.988639\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.990629\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.993262\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.995084\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.997308\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.999692\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.001244\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.003164\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.004509\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.006500\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.375412\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.375\n \"ts\"=\"2026-04-10 01:42:27.375486\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\n STEP: creating customer resources @ 04/10/26 01:42:31.654\n STEP: creating the cluster @ 04/10/26 01:43:40.574\n \"ts\"=\"2026-04-10 01:43:40.574885\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"autoscaling-hcp-cluster\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.976307\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.978331\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.979738\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.982005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.984519\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.986359\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.988639\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.990629\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.993262\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.995084\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.997308\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.999692\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.001244\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.003164\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.004509\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.006500\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.375412\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.375\n \"ts\"=\"2026-04-10 01:42:27.375486\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\n STEP: creating customer resources @ 04/10/26 01:42:31.654\n STEP: creating the cluster @ 04/10/26 01:43:40.574\n \"ts\"=\"2026-04-10 01:43:40.574885\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"autoscaling-hcp-cluster\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","signed\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.970126\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.972003\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.974055\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.976115\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.978443\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.980652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.983036\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.985005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.986981\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.988915\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.990940\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.992341\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:25.994468\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:25.995916\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:25.999219\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.001652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.362711\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.362\n \"ts\"=\"2026-04-10 01:42:27.362860\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/10/26 01:42:31.242\n STEP: creating a standard hcp cluster @ 04/10/26 01:43:33.26\n \"ts\"=\"2026-04-10 01:43:33.260551\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"tls-endpoint-hcp-cluster\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nsigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.970126\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.972003\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.974055\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.976115\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.978443\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.980652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.983036\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.985005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.986981\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.988915\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.990940\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.992341\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:25.994468\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:25.995916\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:25.999219\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.001652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.362711\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.362\n \"ts\"=\"2026-04-10 01:42:27.362860\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/10/26 01:42:31.242\n STEP: creating a standard hcp cluster @ 04/10/26 01:43:33.26\n \"ts\"=\"2026-04-10 01:43:33.260551\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"tls-endpoint-hcp-cluster\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"144724c9be25fac726ad599942218985e7cb1c767a58c2aaf40d3e1a56b3670d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"25112cf607019a413ef4df94318c39f9c02def0919498039896d48d73186ae50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"947d109d8d43c9f5ce0a3ad65337cbc76c5e5f47e6a6e8e25fe216f7aba3aba0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"9b8dfe83a7175b3c3ff1dfc846c5bcb0af067c5e163ffa5144f809ba04cb7fac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"a44703a6832a34a86e069e863ce95031cec6241d092adc5005108861b3a236fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"bcf9a78e2f0d6aacfd2a03648c0125ea09a1d71a284597b72aa79e4ee078e01f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"cf618007959110b43c9382ce0265e98bf5e4ae17099efc8020ed34e576a1d184","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"d26642690096c377827f89a1b690bbd6e91c6a40cad085c73f784a6bea3470e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e7b16355619287ee49663b13c0880b88572d6800c4fb83e0f33d39be07eaf924","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"f4da3c4b9ddd4ac2311fdb82b1a39837f9b3df6336381691c7f2b8bed4807b49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"0a2db9b0c728d07e9e80df888d3f0ca03fe60b12602e85ef9065fba88fc78ab8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"35df135bd95fbdb3da8af53544341ab557e41e5de58dcd707fcc3fd1aa7d8ba8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"389055155e0e9e664a2b8bbcab0e62ebe98717b50141fe5e2fc8bd747cb12977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"905b4519fd2ae631d09bec2ad336fe05d492e64ad60a1c4840fb63249115793c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"0e3db37633accfbbb3e81a3af573a728d2faa0040f7694a8ebd9aa388f5fbf5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"0fb0bab9c886e219989c17756195bd6d490d3111d3d7e44bba6ea3fa77bb66c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"809d7844f1a48c70c3423a63cecd4101fb58ab0d6a342e98acba6ff46c3c4077","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"b5735b60f9dfe41ca07bdf25a9ce5b92363a013abfc259e91caa0c98738482cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"f2738bf06536a42aa77a9502fb7deb243a43c8d2c689ef96192d23fe73aa8ea4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":5,"after_last_push_count":24,"trend_counts":[7,0,3,1,1,0,5],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":10,"prior_last_seen_at":"2026-04-05T20:06:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001418660\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-k2bkdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001418660\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-k2bkdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:165]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4e0c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-vpz2m5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:165]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4e0c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-vpz2m5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016011a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2668zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-wkpcsh/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016011a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2668zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-wkpcsh/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"288c8d145446a41254ecea6baff1c303123323830880407686bcc2a82988756f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"68b4f79c4388b2a38f2fc7f55a3630604102f569c8dd5b906c95a4901a1ac61c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"97312cc094263b3ba5463d178634f674da6726b6e062cecc0738c80b9e6d6fbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"aa2f8fa3e1bb2876e9c5056d271c2441da23870796e6387dd2d2709f791ed027","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"c44640292de911bca3ae3026f422b2d682df399df9a22d6e4536f3e7e6da6324","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":60,"trend_counts":[8,0,9,7,9,3,4],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-09","2026-03-16","2026-03-30"],"prior_runs_affected":37,"prior_last_seen_at":"2026-04-05T18:04:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":13,"prior_last_seen_at":"2026-04-03T18:45:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000620120\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2r9cp8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2r9cp8, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000620120\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2r9cp8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2r9cp8, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b03170\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b03170\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001578ae0\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7nb8nk\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7nb8nk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001578ae0\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7nb8nk\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7nb8nk, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"a0eb876f1dcc8d31439a1e9ac1c75b37fa3dc26ff9193f4bfa7d679cba33c237","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"01edd05054d3d9450ab30aa909069d2d80dc296ce304743832a27fede9a35a05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"7843d873e84cfde4df6aa09b0a6bbf5526a5df8bc96e125218f65bb90b5c80b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"95ffaf23bec65ec54f5a233ebbd9ba6ee14445df48b2a02c2a9c8c45aa82d42e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":33.333333333333336,"anchor_occurrences":3,"after_last_push_count":16,"trend_counts":[2,0,1,1,5,0,3],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":16,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":3}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T01:45:22Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-09T04:15:36Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-09T04:05:03Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-26T02:06:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:168]: Unexpected error:\n \u003c*url.Error | 0xc001248ab0\u003e: \n Get \"https://api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.q6i1z1a9v3j3w8r.qgw8.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:168]: Unexpected error:\n \u003c*url.Error | 0xc001248ab0\u003e: \n Get \"https://api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.q6i1z1a9v3j3w8r.qgw8.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc0013be210\u003e: \n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyBasicAccess failed: failed to list services: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc0013be210\u003e: \n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyBasicAccess failed: failed to list services: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"2769ee201200e30cdfaa3da531246e1754a9bebacbf8e5667de0496fbc64cba4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"691bb7eb88df394a0a911b08820aeb36de06814b9281babcaddd7c570f135e34","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"683173d3a6520478fe76027389479002ac2d2e1044c6fdc87add183d29683bd7","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:141]: Unexpected error:\n \u003c*exported.ResponseError | 0xc001404990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rclvn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:141]: Unexpected error:\n \u003c*exported.ResponseError | 0xc001404990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rclvn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"87ddbde6dce1408bf8bb9c3fe0aef6b8db9623c90da591e311e5dcaaae1ec7c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"635a2afde9e3afd24ee6abfeb373ae222c4b51e9e9a5fed30800c3f45ff07db5","failure_pattern":"get deployment router-default -n openshift-ingress: Get \"\u003curl\u003e","search_query":"get deployment router-default -n openshift-ingress: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-03..2026-04-09 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.000s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d582a0\u003e: \n get deployment router-default -n openshift-ingress: Get \"https://api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io:443/apis/apps/v1/namespaces/openshift-ingress/deployments/router-default\": tls: failed to verify certificate: x509: certificate is valid for api.o6n1s5y5u2a8q7c.wjok.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.000s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d582a0\u003e: \n get deployment router-default -n openshift-ingress: Get \"https://api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io:443/apis/apps/v1/namespaces/openshift-ingress/deployments/router-default\": tls: failed to verify certificate: x509: certificate is valid for api.o6n1s5y5u2a8q7c.wjok.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"039071a1f2cdd978847b2c39b4ab9740ad502f6c020afd94bf765b938f2cfe82","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}