{"meta":{"start_date":"2026-04-27","end_date":"2026-05-03","start_at":"2026-04-27T00:00:00Z","end_at":"2026-05-04T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T12:48:02Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":32,"failed_runs":4,"raw_occurrences":22,"matched_occurrences":21,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"225da8688b41396aa48aa68c786d72b0fb21a4c086f2227d5b5aca2f9f20ebe5","failure_pattern":"failed to get latest install version for candidate channel: query candidate graph for \u003cocp-channel\u003e: Get \"\u003curl\u003e\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,9,0,0],"trend_range":"2026-04-26..2026-05-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:127]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:127]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20: Get \"https://api.openshift.com/api/upgrades_info/v1/graph?channel=candidate-4.20\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"a5ecc4176a5d6acfdaab61e279d004441af852c5ecbbe8365a6be5b677958fbb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"2262aa6e0bdf0119c898aa72ff4ddeaefa91b26c822d66763083a88fd15b5f18","failure_pattern":"failed to get latest install version for candidate channel: query candidate graph for \u003cocp-channel\u003e returned 503 Service Unavailable: upstream connect error or disconnect/reset before headers. reset reason: overflow","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: upstream connect error or disconnect/reset before headers. reset reason: overflow","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,4,0,0],"trend_range":"2026-04-26..2026-05-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: upstream connect error or disconnect/reset before headers. reset reason: overflow\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: upstream connect error or disconnect/reset before headers. reset reason: overflow"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"72335fb1733cca9e495c93f3f113ed7706a9679621e85dc8cf0ed8efaddc0a71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"72335fb1733cca9e495c93f3f113ed7706a9679621e85dc8cf0ed8efaddc0a71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"72335fb1733cca9e495c93f3f113ed7706a9679621e85dc8cf0ed8efaddc0a71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"72335fb1733cca9e495c93f3f113ed7706a9679621e85dc8cf0ed8efaddc0a71","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8743f5d9fceba1a552ae1f0498938ebb795ebcad6b00773daffa79e96dca4ead","failure_pattern":"ResponseFailed: unexpected HTTP status: 504 Gateway Timeout","search_query":"ResponseFailed: unexpected HTTP status: 504 Gateway Timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,3,0,0],"trend_range":"2026-04-26..2026-05-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:64]: Unexpected error:\n    \u003c*cincinnati.Error | 0xc001000000\u003e: \n    ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:64]: Unexpected error:\n    \u003c*cincinnati.Error | 0xc001000000\u003e: \n    ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:76]: Unexpected error:\n    \u003c*cincinnati.Error | 0xc000ee3620\u003e: \n    ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:76]: Unexpected error:\n    \u003c*cincinnati.Error | 0xc000ee3620\u003e: \n    ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:64]: Unexpected error:\n    \u003c*cincinnati.Error | 0xc000e00000\u003e: \n    ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:64]: Unexpected error:\n    \u003c*cincinnati.Error | 0xc000e00000\u003e: \n    ResponseFailed: unexpected HTTP status: 504 Gateway Timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"65efaaa0e95ccddd9858187bb0dabf5e7a838da8dcca75420c9c2e8d3c9ef95a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"c6734f39cfbeb88cc97a458ce36dc5a335b3cf0e560d33d84be914290c1aae27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"de7d915dc89ae397df90ed9f06b0a690dd5bc5dcacec83186a64cbee33946843","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0699bc62b460b38d6978e84316ec365dbd0294b335770344322bfad0fbc5fdf0","failure_pattern":"failed to get latest install version for candidate channel: query candidate graph for \u003cocp-channel\u003e returned 503 Service Unavailable: \u003chtml\u003e","search_query":"failure","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,3,0,0],"trend_range":"2026-04-26..2026-05-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:100]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:127]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:127]: failed to get latest install version for candidate channel: query candidate graph for candidate-4.20 returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"5c4d275c57dbc0f09cd02d94f4d91a04059df336560c8549ec30c893c55ec04f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"5c4d275c57dbc0f09cd02d94f4d91a04059df336560c8549ec30c893c55ec04f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2050174051556528128","occurred_at":"2026-05-01T11:22:38Z","signature_id":"5c4d275c57dbc0f09cd02d94f4d91a04059df336560c8549ec30c893c55ec04f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5srxch, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5srxch\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5srxch, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-04-26..2026-05-03 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-13","2026-04-20"],"prior_runs_affected":11,"prior_last_seen_at":"2026-04-23T21:48:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001140270\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5srxch, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5srxch\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5srxch, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001140270\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5srxch, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5srxch\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5srxch, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2049224514352451584","occurred_at":"2026-04-28T20:29:31Z","signature_id":"de669c49a1fdfba54af97742b26033c993d73cc3add2bd916f8e5dea331ff434","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.125,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-04-26..2026-05-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-13","2026-04-20"],"prior_runs_affected":3,"prior_last_seen_at":"2026-04-21T01:23:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0006dc980\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-jwljr.apps.aro.c6o3e7z9u4k1b7d.wodt.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0006dc980\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-jwljr.apps.aro.c6o3e7z9u4k1b7d.wodt.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2049630553388355584","occurred_at":"2026-04-29T23:22:58Z","signature_id":"6a49f8d689adefdf9782d28d4b02801f077f78b852ffa0709c638c16404a5b4a","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}