From 60de0a9091072f193074a74cf03e957b2f65066d Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Fri, 12 May 2023 10:25:47 -0400 Subject: [PATCH] no bug - Configure CircleCI to break the selenium UI tests up to drastically decrease total run time --- .circleci/config.yml | 114 ++++++++++++++++-- ...ation.t => 1_test_bmo_autolinkification.t} | 0 ...r_new_bug.t => 1_test_bmo_enter_new_bug.t} | 0 ...re_values.t => 1_test_bmo_retire_values.t} | 0 qa/t/{test_bug_edit.t => 1_test_bug_edit.t} | 0 ...se_priority.t => 1_test_choose_priority.t} | 0 ...sifications.t => 1_test_classifications.t} | 0 qa/t/{test_config.t => 1_test_config.t} | 0 ...counts.t => 1_test_create_user_accounts.t} | 0 ...custom_fields.t => 1_test_custom_fields.t} | 0 ...s_admin.t => 1_test_custom_fields_admin.t} | 0 ...fault_groups.t => 2_test_default_groups.t} | 0 ...t_dependencies.t => 2_test_dependencies.t} | 0 ...es.t => 2_test_edit_products_properties.t} | 0 ...enter_new_bug.t => 2_test_enter_new_bug.t} | 0 qa/t/{test_flags.t => 2_test_flags.t} | 0 qa/t/{test_flags2.t => 2_test_flags2.t} | 0 qa/t/{test_groups.t => 2_test_groups.t} | 0 qa/t/{test_keywords.t => 2_test_keywords.t} | 0 qa/t/{test_login.t => 2_test_login.t} | 0 ...{test_milestones.t => 2_test_milestones.t} | 0 ...est_new_release.t => 3_test_new_release.t} | 0 ...chments.t => 3_test_private_attachments.t} | 0 ...{test_qa_contact.t => 3_test_qa_contact.t} | 0 ...{test_rate_limit.t => 3_test_rate_limit.t} | 0 ...require_login.t => 3_test_require_login.t} | 0 ...t_sanity_check.t => 3_test_sanity_check.t} | 0 ...ved_searches.t => 3_test_saved_searches.t} | 0 qa/t/{test_search.t => 3_test_search.t} | 0 qa/t/{test_security.t => 3_test_security.t} | 19 +-- qa/t/{test_see_also.t => 3_test_see_also.t} | 0 ...ed_searches.t => 4_test_shared_searches.t} | 0 ..._products.t => 4_test_show_all_products.t} | 0 ...hiteboard.t => 4_test_status_whiteboard.t} | 0 ...sudo_sessions.t => 4_test_sudo_sessions.t} | 0 ...ilestones.t => 4_test_target_milestones.t} | 0 ...t_time_summary.t => 4_test_time_summary.t} | 0 ...est_user_groups.t => 4_test_user_groups.t} | 0 ...user_matching.t => 4_test_user_matching.t} | 0 ...references.t => 4_test_user_preferences.t} | 0 ...{test_user_privs.t => 4_test_user_privs.t} | 0 qa/t/{test_votes.t => 4_test_votes.t} | 0 qa/t/{test_webhooks.t => 4_test_webhooks.t} | 0 scripts/entrypoint.pl | 6 +- 44 files changed, 119 insertions(+), 20 deletions(-) rename qa/t/{test_bmo_autolinkification.t => 1_test_bmo_autolinkification.t} (100%) rename qa/t/{test_bmo_enter_new_bug.t => 1_test_bmo_enter_new_bug.t} (100%) rename qa/t/{test_bmo_retire_values.t => 1_test_bmo_retire_values.t} (100%) rename qa/t/{test_bug_edit.t => 1_test_bug_edit.t} (100%) rename qa/t/{test_choose_priority.t => 1_test_choose_priority.t} (100%) rename qa/t/{test_classifications.t => 1_test_classifications.t} (100%) rename qa/t/{test_config.t => 1_test_config.t} (100%) rename qa/t/{test_create_user_accounts.t => 1_test_create_user_accounts.t} (100%) rename qa/t/{test_custom_fields.t => 1_test_custom_fields.t} (100%) rename qa/t/{test_custom_fields_admin.t => 1_test_custom_fields_admin.t} (100%) rename qa/t/{test_default_groups.t => 2_test_default_groups.t} (100%) rename qa/t/{test_dependencies.t => 2_test_dependencies.t} (100%) rename qa/t/{test_edit_products_properties.t => 2_test_edit_products_properties.t} (100%) rename qa/t/{test_enter_new_bug.t => 2_test_enter_new_bug.t} (100%) rename qa/t/{test_flags.t => 2_test_flags.t} (100%) rename qa/t/{test_flags2.t => 2_test_flags2.t} (100%) rename qa/t/{test_groups.t => 2_test_groups.t} (100%) rename qa/t/{test_keywords.t => 2_test_keywords.t} (100%) rename qa/t/{test_login.t => 2_test_login.t} (100%) rename qa/t/{test_milestones.t => 2_test_milestones.t} (100%) rename qa/t/{test_new_release.t => 3_test_new_release.t} (100%) rename qa/t/{test_private_attachments.t => 3_test_private_attachments.t} (100%) rename qa/t/{test_qa_contact.t => 3_test_qa_contact.t} (100%) rename qa/t/{test_rate_limit.t => 3_test_rate_limit.t} (100%) rename qa/t/{test_require_login.t => 3_test_require_login.t} (100%) rename qa/t/{test_sanity_check.t => 3_test_sanity_check.t} (100%) rename qa/t/{test_saved_searches.t => 3_test_saved_searches.t} (100%) rename qa/t/{test_search.t => 3_test_search.t} (100%) rename qa/t/{test_security.t => 3_test_security.t} (94%) rename qa/t/{test_see_also.t => 3_test_see_also.t} (100%) rename qa/t/{test_shared_searches.t => 4_test_shared_searches.t} (100%) rename qa/t/{test_show_all_products.t => 4_test_show_all_products.t} (100%) rename qa/t/{test_status_whiteboard.t => 4_test_status_whiteboard.t} (100%) rename qa/t/{test_sudo_sessions.t => 4_test_sudo_sessions.t} (100%) rename qa/t/{test_target_milestones.t => 4_test_target_milestones.t} (100%) rename qa/t/{test_time_summary.t => 4_test_time_summary.t} (100%) rename qa/t/{test_user_groups.t => 4_test_user_groups.t} (100%) rename qa/t/{test_user_matching.t => 4_test_user_matching.t} (100%) rename qa/t/{test_user_preferences.t => 4_test_user_preferences.t} (100%) rename qa/t/{test_user_privs.t => 4_test_user_privs.t} (100%) rename qa/t/{test_votes.t => 4_test_votes.t} (100%) rename qa/t/{test_webhooks.t => 4_test_webhooks.t} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd377f26d3..352e66bd5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,7 +171,7 @@ jobs: command: | [[ -f build_info/only_version_changed.txt ]] && exit 0 docker-compose -f docker-compose.test.yml run --no-deps bmo.test \ - test_sanity t/*.t extensions/*/t/*.t | tee artifacts/$CIRCLE_JOB.txt + test_sanity t/*.t extensions/*/t/*.t - store_artifacts: path: /app/artifacts - *store_log @@ -193,15 +193,42 @@ jobs: [[ -d artifacts ]] || mkdir artifacts - attach_workspace: at: /app/build_info + - run: + name: run webservice tests + command: | + [[ -f build_info/only_version_changed.txt ]] && exit 0 + docker-compose -f docker-compose.test.yml run bmo.test test_webservices + - store_artifacts: + path: /app/artifacts + - *store_log + + test_selenium_1: + parallelism: 1 + working_directory: /app + docker: + - *docker_image + steps: + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: true + - checkout + - *install_docker_compose + - *docker_login + - *build_image - run: | - [[ -f build_info/only_version_changed.txt ]] && exit 0 - docker-compose -f docker-compose.test.yml run bmo.test test_webservices | \ - tee artifacts/$CIRCLE_JOB.txt + [[ -d artifacts ]] || mkdir artifacts + - attach_workspace: + at: /app/build_info + - run: + name: run selenium tests + command: | + [[ -f build_info/only_version_changed.txt ]] && exit 0 + docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=1 bmo.test test_selenium - store_artifacts: path: /app/artifacts - *store_log - test_selenium: + test_selenium_2: parallelism: 1 working_directory: /app docker: @@ -218,10 +245,63 @@ jobs: [[ -d artifacts ]] || mkdir artifacts - attach_workspace: at: /app/build_info + - run: + name: run selenium tests + command: | + [[ -f build_info/only_version_changed.txt ]] && exit 0 + docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=2 bmo.test test_selenium 2 + - store_artifacts: + path: /app/artifacts + - *store_log + + test_selenium_3: + parallelism: 1 + working_directory: /app + docker: + - *docker_image + steps: + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: true + - checkout + - *install_docker_compose + - *docker_login + - *build_image - run: | - [[ -f build_info/only_version_changed.txt ]] && exit 0 - docker-compose -f docker-compose.test.yml run bmo.test test_selenium | \ - tee artifacts/$CIRCLE_JOB.txt + [[ -d artifacts ]] || mkdir artifacts + - attach_workspace: + at: /app/build_info + - run: + name: run selenium tests + command: | + [[ -f build_info/only_version_changed.txt ]] && exit 0 + docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=3 bmo.test test_selenium 3 + - store_artifacts: + path: /app/artifacts + - *store_log + + test_selenium_4: + parallelism: 1 + working_directory: /app + docker: + - *docker_image + steps: + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: true + - checkout + - *install_docker_compose + - *docker_login + - *build_image + - run: | + [[ -d artifacts ]] || mkdir artifacts + - attach_workspace: + at: /app/build_info + - run: + name: run selenium tests + command: | + [[ -f build_info/only_version_changed.txt ]] && exit 0 + docker-compose -f docker-compose.test.yml run -e SELENIUM_GROUP=4 bmo.test test_selenium 4 - store_artifacts: path: /app/artifacts - *store_log @@ -248,7 +328,7 @@ jobs: command: | [[ -f build_info/only_version_changed.txt ]] && exit 0 docker-compose -f docker-compose.test.yml run bmo.test test_bmo \ - -q -f t/bmo/*.t extensions/*/t/bmo/*.t | tee artifacts/$CIRCLE_JOB.txt + -q -f t/bmo/*.t extensions/*/t/bmo/*.t - *store_log workflows: @@ -262,7 +342,10 @@ workflows: - test_sanity - test_bmo - test_webservices - - test_selenium + - test_selenium_1 + - test_selenium_2 + - test_selenium_3 + - test_selenium_4 - test_sanity: requires: - build @@ -272,6 +355,15 @@ workflows: - test_webservices: requires: - build - - test_selenium: + - test_selenium_1: + requires: + - build + - test_selenium_2: + requires: + - build + - test_selenium_3: + requires: + - build + - test_selenium_4: requires: - build diff --git a/qa/t/test_bmo_autolinkification.t b/qa/t/1_test_bmo_autolinkification.t similarity index 100% rename from qa/t/test_bmo_autolinkification.t rename to qa/t/1_test_bmo_autolinkification.t diff --git a/qa/t/test_bmo_enter_new_bug.t b/qa/t/1_test_bmo_enter_new_bug.t similarity index 100% rename from qa/t/test_bmo_enter_new_bug.t rename to qa/t/1_test_bmo_enter_new_bug.t diff --git a/qa/t/test_bmo_retire_values.t b/qa/t/1_test_bmo_retire_values.t similarity index 100% rename from qa/t/test_bmo_retire_values.t rename to qa/t/1_test_bmo_retire_values.t diff --git a/qa/t/test_bug_edit.t b/qa/t/1_test_bug_edit.t similarity index 100% rename from qa/t/test_bug_edit.t rename to qa/t/1_test_bug_edit.t diff --git a/qa/t/test_choose_priority.t b/qa/t/1_test_choose_priority.t similarity index 100% rename from qa/t/test_choose_priority.t rename to qa/t/1_test_choose_priority.t diff --git a/qa/t/test_classifications.t b/qa/t/1_test_classifications.t similarity index 100% rename from qa/t/test_classifications.t rename to qa/t/1_test_classifications.t diff --git a/qa/t/test_config.t b/qa/t/1_test_config.t similarity index 100% rename from qa/t/test_config.t rename to qa/t/1_test_config.t diff --git a/qa/t/test_create_user_accounts.t b/qa/t/1_test_create_user_accounts.t similarity index 100% rename from qa/t/test_create_user_accounts.t rename to qa/t/1_test_create_user_accounts.t diff --git a/qa/t/test_custom_fields.t b/qa/t/1_test_custom_fields.t similarity index 100% rename from qa/t/test_custom_fields.t rename to qa/t/1_test_custom_fields.t diff --git a/qa/t/test_custom_fields_admin.t b/qa/t/1_test_custom_fields_admin.t similarity index 100% rename from qa/t/test_custom_fields_admin.t rename to qa/t/1_test_custom_fields_admin.t diff --git a/qa/t/test_default_groups.t b/qa/t/2_test_default_groups.t similarity index 100% rename from qa/t/test_default_groups.t rename to qa/t/2_test_default_groups.t diff --git a/qa/t/test_dependencies.t b/qa/t/2_test_dependencies.t similarity index 100% rename from qa/t/test_dependencies.t rename to qa/t/2_test_dependencies.t diff --git a/qa/t/test_edit_products_properties.t b/qa/t/2_test_edit_products_properties.t similarity index 100% rename from qa/t/test_edit_products_properties.t rename to qa/t/2_test_edit_products_properties.t diff --git a/qa/t/test_enter_new_bug.t b/qa/t/2_test_enter_new_bug.t similarity index 100% rename from qa/t/test_enter_new_bug.t rename to qa/t/2_test_enter_new_bug.t diff --git a/qa/t/test_flags.t b/qa/t/2_test_flags.t similarity index 100% rename from qa/t/test_flags.t rename to qa/t/2_test_flags.t diff --git a/qa/t/test_flags2.t b/qa/t/2_test_flags2.t similarity index 100% rename from qa/t/test_flags2.t rename to qa/t/2_test_flags2.t diff --git a/qa/t/test_groups.t b/qa/t/2_test_groups.t similarity index 100% rename from qa/t/test_groups.t rename to qa/t/2_test_groups.t diff --git a/qa/t/test_keywords.t b/qa/t/2_test_keywords.t similarity index 100% rename from qa/t/test_keywords.t rename to qa/t/2_test_keywords.t diff --git a/qa/t/test_login.t b/qa/t/2_test_login.t similarity index 100% rename from qa/t/test_login.t rename to qa/t/2_test_login.t diff --git a/qa/t/test_milestones.t b/qa/t/2_test_milestones.t similarity index 100% rename from qa/t/test_milestones.t rename to qa/t/2_test_milestones.t diff --git a/qa/t/test_new_release.t b/qa/t/3_test_new_release.t similarity index 100% rename from qa/t/test_new_release.t rename to qa/t/3_test_new_release.t diff --git a/qa/t/test_private_attachments.t b/qa/t/3_test_private_attachments.t similarity index 100% rename from qa/t/test_private_attachments.t rename to qa/t/3_test_private_attachments.t diff --git a/qa/t/test_qa_contact.t b/qa/t/3_test_qa_contact.t similarity index 100% rename from qa/t/test_qa_contact.t rename to qa/t/3_test_qa_contact.t diff --git a/qa/t/test_rate_limit.t b/qa/t/3_test_rate_limit.t similarity index 100% rename from qa/t/test_rate_limit.t rename to qa/t/3_test_rate_limit.t diff --git a/qa/t/test_require_login.t b/qa/t/3_test_require_login.t similarity index 100% rename from qa/t/test_require_login.t rename to qa/t/3_test_require_login.t diff --git a/qa/t/test_sanity_check.t b/qa/t/3_test_sanity_check.t similarity index 100% rename from qa/t/test_sanity_check.t rename to qa/t/3_test_sanity_check.t diff --git a/qa/t/test_saved_searches.t b/qa/t/3_test_saved_searches.t similarity index 100% rename from qa/t/test_saved_searches.t rename to qa/t/3_test_saved_searches.t diff --git a/qa/t/test_search.t b/qa/t/3_test_search.t similarity index 100% rename from qa/t/test_search.t rename to qa/t/3_test_search.t diff --git a/qa/t/test_security.t b/qa/t/3_test_security.t similarity index 94% rename from qa/t/test_security.t rename to qa/t/3_test_security.t index 2cd85f478b..5b60c68c46 100644 --- a/qa/t/test_security.t +++ b/qa/t/3_test_security.t @@ -132,22 +132,25 @@ $sel->check_ok('//input[@name="groups" and @value="Master"]'); my $bug2_id = create_bug($sel, $bug_summary2); go_to_bug($sel, $bug1_id); -$sel->is_text_present_ok("secret_qa_bug_$bug2_id"); +$sel->is_element_present_ok("link=secret_qa_bug_$bug2_id"); logout($sel); log_in($sel, $config, 'editbugs'); go_to_bug($sel, $bug1_id); -ok(!$sel->is_text_present("secret_qa_bug_$bug2_id"), - "The alias 'secret_qa_bug_$bug2_id' is not visible for editbugs users"); -ok($sel->is_text_present($bug2_id), - "But the bug ID is visible for editbugs users"); +ok( + !$sel->is_element_present("link=secret_qa_bug_$bug2_id"), + "The alias 'secret_qa_bug_$bug2_id' is not visible for editbugs users" +); +$sel->is_element_present_ok("link=$bug2_id"); logout($sel); go_to_bug($sel, $bug1_id, 1); -ok(!$sel->is_text_present("secret_qa_bug_$bug2_id"), +ok(!$sel->is_element_present("link=secret_qa_bug_$bug2_id"), "The alias 'secret_qa_bug_$bug2_id' is not visible for logged out users"); -ok(!$sel->is_text_present($bug2_id), - "Even the bug ID is not visible for logged out users"); +ok( + !$sel->is_element_present("link=$bug2_id"), + "Even the bug ID is not visible for logged out users" +); ####################################################################### # Security bug 472206. diff --git a/qa/t/test_see_also.t b/qa/t/3_test_see_also.t similarity index 100% rename from qa/t/test_see_also.t rename to qa/t/3_test_see_also.t diff --git a/qa/t/test_shared_searches.t b/qa/t/4_test_shared_searches.t similarity index 100% rename from qa/t/test_shared_searches.t rename to qa/t/4_test_shared_searches.t diff --git a/qa/t/test_show_all_products.t b/qa/t/4_test_show_all_products.t similarity index 100% rename from qa/t/test_show_all_products.t rename to qa/t/4_test_show_all_products.t diff --git a/qa/t/test_status_whiteboard.t b/qa/t/4_test_status_whiteboard.t similarity index 100% rename from qa/t/test_status_whiteboard.t rename to qa/t/4_test_status_whiteboard.t diff --git a/qa/t/test_sudo_sessions.t b/qa/t/4_test_sudo_sessions.t similarity index 100% rename from qa/t/test_sudo_sessions.t rename to qa/t/4_test_sudo_sessions.t diff --git a/qa/t/test_target_milestones.t b/qa/t/4_test_target_milestones.t similarity index 100% rename from qa/t/test_target_milestones.t rename to qa/t/4_test_target_milestones.t diff --git a/qa/t/test_time_summary.t b/qa/t/4_test_time_summary.t similarity index 100% rename from qa/t/test_time_summary.t rename to qa/t/4_test_time_summary.t diff --git a/qa/t/test_user_groups.t b/qa/t/4_test_user_groups.t similarity index 100% rename from qa/t/test_user_groups.t rename to qa/t/4_test_user_groups.t diff --git a/qa/t/test_user_matching.t b/qa/t/4_test_user_matching.t similarity index 100% rename from qa/t/test_user_matching.t rename to qa/t/4_test_user_matching.t diff --git a/qa/t/test_user_preferences.t b/qa/t/4_test_user_preferences.t similarity index 100% rename from qa/t/test_user_preferences.t rename to qa/t/4_test_user_preferences.t diff --git a/qa/t/test_user_privs.t b/qa/t/4_test_user_privs.t similarity index 100% rename from qa/t/test_user_privs.t rename to qa/t/4_test_user_privs.t diff --git a/qa/t/test_votes.t b/qa/t/4_test_votes.t similarity index 100% rename from qa/t/test_votes.t rename to qa/t/4_test_votes.t diff --git a/qa/t/test_webhooks.t b/qa/t/4_test_webhooks.t similarity index 100% rename from qa/t/test_webhooks.t rename to qa/t/4_test_webhooks.t diff --git a/scripts/entrypoint.pl b/scripts/entrypoint.pl index e64ef3eb01..66c5dfcb7b 100755 --- a/scripts/entrypoint.pl +++ b/scripts/entrypoint.pl @@ -165,7 +165,11 @@ sub cmd_test_webservices { } sub cmd_test_selenium { - cmd_test_qa('test_*.t'); + my $file_pattern = '*test_*.t'; + if ($ENV{SELENIUM_GROUP}) { + $file_pattern = $ENV{SELENIUM_GROUP} . '_test_*.t'; + } + cmd_test_qa($file_pattern); } sub cmd_test_qa {