{"meta":{"start_date":"2026-05-27","end_date":"2026-05-27","start_at":"2026-05-27T00:00:00Z","end_at":"2026-05-28T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T13:39:37Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":3,"raw_occurrences":5,"matched_occurrences":4,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"66b5f9093b7c3631a8a6fa5876783cc486e1a8c41685c319c9cd74e2db5060e7","failure_pattern":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: test pods not yet ready: [echo-b-db89d9c95-khrvb (phase: Pending) echo-b-host-95cbd8655-ntvsj (phase: Pending)","search_query":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: test pods not yet ready: [echo-b-db89d9c95-khrvb (phase: Pending) echo-b-host-95cbd8655-ntvsj (phase: Pending) host-to-b-multi-node-clusterip-c54bf67bf-z8w7b (container host-to-b-multi-node-clusterip-container waiting: CrashLoopBackOff) host-to-b-multi-node-headless-55f66fc4c7-t4gk2 (container host-to-b-multi-node-headless-container waiting: CrashLoopBackOff) pod-to-b-intra-node-nodeport-5884978697-tfxc6 (phase: Pending) pod-to-b-multi-node-clusterip-7d65578cf5-b5f6m (container pod-to-b-multi-node-clusterip-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8557d86d6f-2gkgv (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff) pod-to-b-multi-node-nodeport-7847b5df8f-q47xx (container pod-to-b-multi-node-nodeport-container waiting: CrashLoopBackOff)]","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-21..2026-05-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:211]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f1ef90\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: test pods not yet ready: [echo-b-db89d9c95-khrvb (phase: Pending) echo-b-host-95cbd8655-ntvsj (phase: Pending) host-to-b-multi-node-clusterip-c54bf67bf-z8w7b (container host-to-b-multi-node-clusterip-container waiting: CrashLoopBackOff) host-to-b-multi-node-headless-55f66fc4c7-t4gk2 (container host-to-b-multi-node-headless-container waiting: CrashLoopBackOff) pod-to-b-intra-node-nodeport-5884978697-tfxc6 (phase: Pending) pod-to-b-multi-node-clusterip-7d65578cf5-b5f6m (container pod-to-b-multi-node-clusterip-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8557d86d6f-2gkgv (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff) pod-to-b-multi-node-nodeport-7847b5df8f-q47xx (container pod-to-b-multi-node-nodeport-container waiting: CrashLoopBackOff)]\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:211]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f1ef90\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: test pods not yet ready: [echo-b-db89d9c95-khrvb (phase: Pending) echo-b-host-95cbd8655-ntvsj (phase: Pending) host-to-b-multi-node-clusterip-c54bf67bf-z8w7b (container host-to-b-multi-node-clusterip-container waiting: CrashLoopBackOff) host-to-b-multi-node-headless-55f66fc4c7-t4gk2 (container host-to-b-multi-node-headless-container waiting: CrashLoopBackOff) pod-to-b-intra-node-nodeport-5884978697-tfxc6 (phase: Pending) pod-to-b-multi-node-clusterip-7d65578cf5-b5f6m (container pod-to-b-multi-node-clusterip-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8557d86d6f-2gkgv (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff) pod-to-b-multi-node-nodeport-7847b5df8f-q47xx (container pod-to-b-multi-node-nodeport-container waiting: CrashLoopBackOff)]\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2059472048266153984","occurred_at":"2026-05-27T03:09:34Z","signature_id":"dfd56d0c23391ba9e3f47bc8d24eb26853d056b826c5062597dfd1fc149a86ac","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e457d9ded8188e2e5d3bf94632039456ab0d3f40aab17c66a163827314f30476","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-21..2026-05-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:211]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fd75c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:211]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fd75c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check namespace are ready: failed to list pods in cilium-connectivity-check namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2059569820239663104","occurred_at":"2026-05-27T09:38:04Z","signature_id":"476b2811dc637bf5ee4ed1e4760f2064680823be58a4aaf33282a02b56be4897","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"5cbf867628807a0b12c8830e9f554863db910ef2338615837e07e1e952942010","failure_pattern":"configmaps \"image-registry-allowlist-config\" is forbidden: User \"system:serviceaccount:ci-op-2n8rcldp:integration-e2e-parallel\" cannot get resource \"configmaps\" in API group \"\" in the namespace \"\u003cnamespace\u003e\"","search_query":"configmaps \"image-registry-allowlist-config\" is forbidden: User \"system:serviceaccount:ci-op-2n8rcldp:integration-e2e-parallel\" cannot get resource \"configmaps\" in API group \"\" in the namespace \"image-registry-policy\"","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Image Registry Policy should allow pods with images from allowed registries and have a valid allowlist","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-21..2026-05-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Image Registry Policy should allow pods with images from allowed registries and have a valid allowlist","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:144]: ConfigMap image-registry-allowlist-config should exist\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000e48780\u003e: \n    configmaps \"image-registry-allowlist-config\" is forbidden: User \"system:serviceaccount:ci-op-2n8rcldp:integration-e2e-parallel\" cannot get resource \"configmaps\" in API group \"\" in the namespace \"image-registry-policy\"\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:144]: ConfigMap image-registry-allowlist-config should exist\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000e48780\u003e: \n    configmaps \"image-registry-allowlist-config\" is forbidden: User \"system:serviceaccount:ci-op-2n8rcldp:integration-e2e-parallel\" cannot get resource \"configmaps\" in API group \"\" in the namespace \"image-registry-policy\"\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2059472048266153984","occurred_at":"2026-05-27T03:09:34Z","signature_id":"1b95ca959ea9a86a1c4c4df8ffe31ea12bc9747c6c98ff66093bfc5f8375e86e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gjxj7m\" in resourcegroup=\"rg-np-skip-minor-gjxj7m-vmcss2\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-21..2026-05-27 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-11","2026-05-18"],"prior_runs_affected":11,"prior_last_seen_at":"2026-05-19T20:38:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:278] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001476348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gjxj7m\" in resourcegroup=\"rg-np-skip-minor-gjxj7m-vmcss2\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:278] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001476348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gjxj7m\" in resourcegroup=\"rg-np-skip-minor-gjxj7m-vmcss2\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2059472048266153984","occurred_at":"2026-05-27T03:09:34Z","signature_id":"c665ec1b3959e77c7b04783caf4348d5865b060e8990dde30e55bb6fe7d3c296","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}