diff --git a/icpc-wf/ansible/common_tasks_build.yml b/icpc-wf/ansible/common_tasks_build.yml index d39fc853..252e2511 100644 --- a/icpc-wf/ansible/common_tasks_build.yml +++ b/icpc-wf/ansible/common_tasks_build.yml @@ -3,7 +3,7 @@ - name: run maintainer-conf become: yes become_user: domjudge - command: make maintainer-conf CONFIGURE_FLAGS='--disable-doc-build' + command: make maintainer-conf CONFIGURE_FLAGS='--disable-doc-build --with-judgehost_chrootdir=/chroot/domjudge-systest --with-baseurl=https://systest.domjudge.org/' register: dj_configured args: chdir: "{{DJ_DIR}}" @@ -23,5 +23,5 @@ shell: make -C {{DJ_DIR}} maintainer-postinstall-permissions - name: copy domjudge-sudoers file - copy: remote_src=True src={{DJ_DIR}}/etc/sudoers-domjudge dest=/etc/sudoers.d/domjudge mode=0440 owner=root group=root + copy: remote_src=True src={{DJ_DIR}}/etc/sudoers-domjudge dest=/etc/sudoers.d/domjudge-systest mode=0440 owner=root group=root diff --git a/icpc-wf/ansible/common_tasks_prebuild.yml b/icpc-wf/ansible/common_tasks_prebuild.yml index ed2fa40a..a321ee67 100644 --- a/icpc-wf/ansible/common_tasks_prebuild.yml +++ b/icpc-wf/ansible/common_tasks_prebuild.yml @@ -1,17 +1,5 @@ # Common tasks before building DOMjudge. --- - - name: add domjudge to hosts file - lineinfile: - dest: /etc/hosts - regexp: 'domserver$' - line: "{{DOMSERVER_IP}} domserver" - - - name: set timezone - timezone: - name: "{{TIMEZONE}}" - - - include: common_tasks_packages_icpc-wf.yml - - name: install common required/useful packages tags: packages apt: pkg={{item}} state=present @@ -35,6 +23,7 @@ - php-xml - php-zip - php-mbstring + - php-intl - bsdmainutils - libcgroup-dev - libcurl4-gnutls-dev @@ -83,13 +72,6 @@ - { name: 'email', value: 'team@domjudge.org' } - { name: 'name', value: 'DOMjudge team' } - - name: Allow 'sudo' group to have passwordless sudo - lineinfile: - dest: /etc/sudoers - state: present - regexp: '^%sudo' - line: '%sudo ALL=(ALL) NOPASSWD: ALL' - - name: Create .ssh directory file: path="/home/domjudge/.ssh" group=domjudge owner=domjudge mode=0700 state=directory @@ -151,57 +133,3 @@ - name: configure domjudge logrotate copy: src=files/logrotate.domjudge dest=/etc/logrotate.d/domjudge - - - name: copy DOMjudge logo binary - copy: src=files/domlogo dest=/home/domjudge/domlogo owner=domjudge group=domjudge mode=0755 - - - name: make sure lightdm config directory exists - file: path=/etc/lightdm/lightdm.conf.d state=directory - - - name: enable GDM autologin - lineinfile: - path: /etc/gdm3/custom.conf - regexp: 'AutomaticLoginEnable' - line: 'AutomaticLoginEnable=true' - notify: restart gdm - - - name: Automatically login domjudge user - lineinfile: - path: /etc/gdm3/custom.conf - regexp: 'AutomaticLogin' - line: 'AutomaticLogin=domjudge' - notify: restart gdm - - - name: make sure autostart directory exists - file: dest=/home/domjudge/.config/autostart state=directory owner=domjudge group=domjudge - tags: fix_autostart - - - name: install SSL server certificates - copy: - src: "{{ item }}" - dest: /etc/ssl/certs/ - owner: root - group: root - mode: 0644 - with_fileglob: - - files/ssl/*.crt - notify: update-ca-certificates - - - name: create ca certificates shared directory - file: - dest: /usr/local/share/ca-certificates - state: directory - owner: root - group: root - - - name: install SSL server certificates into CA certificates - copy: - src: "{{ item }}" - dest: /usr/local/share/ca-certificates - owner: root - group: root - mode: 0644 - with_fileglob: - - files/ssl/*.crt - notify: update-ca-certificates - diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 9111c5ef..3925d3bf 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -20,8 +20,6 @@ service: name=rsyslog enabled=yes state=restarted - name: restart systemctl shell: systemctl daemon-reload - - name: restart gdm - service: name=gdm3 enabled=yes state=restarted tasks: - name: include global variables include_vars: variables.yml @@ -31,7 +29,7 @@ - name: install domserver required packages apt: pkg={{item}} state=present with_items: - - mysql-server + - mariadb-server - nginx - php-fpm - python-mysqldb @@ -121,23 +119,12 @@ dest: "{{DJ_DIR}}/webapp/web/style.css" when: BACKGROUND_COLOR is defined - - name: install SSL private key files - copy: - src: "{{ item }}" - dest: /etc/ssl/private/ - owner: root - group: root - mode: 0600 - with_fileglob: - - files/ssl/*.key - notify: update-ca-certificates - - name: copy in domjudge FPM conf copy: src={{DJ_DIR}}/etc/domjudge-fpm.conf remote_src=yes dest=/etc/php/7.2/fpm/pool.d/domjudge.conf notify: restart PHP FPM - name: copy in domjudge nginx conf - copy: src={{DJ_DIR}}/etc/nginx-conf remote_src=yes dest=/etc/nginx/sites-available/domjudge.conf + copy: src={{DJ_DIR}}/etc/nginx-conf remote_src=yes dest=/etc/nginx/sites-available/domjudge-systest notify: restart nginx - name: copy in domjudge inner nginx conf @@ -146,7 +133,7 @@ - name: remove HTTP host blockinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest marker: "{mark}" marker_begin: "### http host config ###" marker_end: "# Alternatively, use HTTPS and redirect HTTP to HTTPS:" @@ -154,29 +141,44 @@ - name: enable HTTP redirect and HTTPS blocks replace: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest after: 'upstream domjudge \{' regexp: '^# ?(.*)' replace: '\1' notify: restart nginx - - name: remove IPv6 listens + - name: remove HTTP redirect host + blockinfile: + path: /etc/nginx/sites-available/domjudge-systest + marker: "{mark}" + marker_begin: " listen 80;" + marker_end: "server {" + notify: restart nginx + + - name: rename upstream lineinfile: - path: /etc/nginx/sites-available/domjudge.conf - regexp: 'listen\s+\[.*\]:\d+;' - state: absent + path: /etc/nginx/sites-available/domjudge-systest + regexp: '^upstream' + line: "upstream domjudgesystest {" + notify: restart nginx + + - name: Set system test hostname + lineinfile: + path: /etc/nginx/snippets/domjudge-inner + regexp: 'server_name' + line: "server_name systest.domjudge.org;" notify: restart nginx - - name: change IPv4 HTTP listen to default server + - name: remove IPv6 listens lineinfile: - path: /etc/nginx/sites-available/domjudge.conf - regexp: 'listen.*80;' - line: "\tlisten 80 default_server;" + path: /etc/nginx/sites-available/domjudge-systest + regexp: 'listen\s+\[.*\]:\d+;' + state: absent notify: restart nginx - name: change IPv4 HTTPS listen to all interfaces lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'listen.*443;' line: "\tlisten 443 ssl http2 default_server;" notify: restart nginx @@ -212,52 +214,50 @@ line: "set $prefix '';" notify: restart nginx + - name: rename fastcgi pass + lineinfile: + path: /etc/nginx/snippets/domjudge-inner + regexp: 'fastcgi_pass' + line: "\tfastcgi_pass domjudgesystest;" + notify: restart nginx + - name: configure SSL certificate lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'ssl_certificate (.*)' line: "\tssl_certificate {{DOMSERVER_SSL_CERT}};" notify: restart nginx - name: configure SSL key lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'ssl_certificate_key (.*)' line: "\tssl_certificate_key {{DOMSERVER_SSL_KEY}};" notify: restart nginx - name: use our own inner nginx configuration lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'include (.*)nginx-conf-inner;' line: "\tinclude /etc/nginx/snippets/domjudge-inner;" notify: restart nginx - name: enable nginx conf for domjudge - file: src=/etc/nginx/sites-available/domjudge.conf dest=/etc/nginx/sites-enabled/domjudge.conf state=link + file: src=/etc/nginx/sites-available/domjudge-systest dest=/etc/nginx/sites-enabled/domjudge-systest state=link notify: restart nginx - name: disable default nginx site file: state=absent path=/etc/nginx/sites-enabled/default notify: restart nginx - - name: Increase PM max children for PHP FPM + - name: Set PHP settings lineinfile: dest: /etc/php/7.2/fpm/pool.d/domjudge.conf - regexp: '^pm\.max_children' - line: 'pm.max_children = 300' + regexp: "{{item.regexp}}" + line: "{{item.key}} = {{item.value}}" with_items: - { key: 'pm.max_children', regexp: '^pm\.max_children', value: '{{PHP_FPM_MAX_CHILDREN}}' } - { key: 'php_admin_value[memory_limit]', regexp: '^php_admin_value\[memory_limit\]', value: '{{PHP_MEMORY_LIMIT}}' } - { key: 'php_admin_value[upload_max_filesize]', regexp: '^php_admin_value\[upload_max_filesize\]', value: '{{PHP_UPLOAD_MAX_FILESIZE}}' } - { key: 'php_admin_value[post_max_size]', regexp: '^php_admin_value\[post_max_size\]', value: '{{PHP_POST_MAX_SIZE}}' } notify: restart PHP FPM - - - name: add autostart shortcuts - copy: src=files/{{item}}.desktop dest=/home/domjudge/.config/autostart/ owner=domjudge group=domjudge mode=0755 - with_items: - - htop - - taillog-domserver-nginx-error - - taillog-domserver-symfony-error - - domjudgelogo-domserver - diff --git a/icpc-wf/ansible/files/create_cgroups.service b/icpc-wf/ansible/files/create_cgroups.service index 36031fba..b2f6fd8c 100644 --- a/icpc-wf/ansible/files/create_cgroups.service +++ b/icpc-wf/ansible/files/create_cgroups.service @@ -3,5 +3,5 @@ Description=Make sure cgroups exist for domjudge judgedaemon [Service] Type=oneshot -ExecStart=/home/domjudge/domjudge/bin/create_cgroups +ExecStart=/opt/domjudge-systest/bin/create_cgroups RemainAfterExit=true diff --git a/icpc-wf/ansible/files/judgedaemon.service b/icpc-wf/ansible/files/judgedaemon0.service similarity index 68% rename from icpc-wf/ansible/files/judgedaemon.service rename to icpc-wf/ansible/files/judgedaemon0.service index ac3176bb..836d4f9c 100644 --- a/icpc-wf/ansible/files/judgedaemon.service +++ b/icpc-wf/ansible/files/judgedaemon0.service @@ -1,15 +1,13 @@ [Unit] Description=DOMjudge JudgeDaemon +After=network.target Requires=create_cgroups.service -Requires=tune_cpu.service After=create_cgroups.service -After=tune_cpu.service -After=network.target [Service] Type=simple -ExecStart=/home/domjudge/domjudge/bin/judgedaemon -n 0 +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 0 User=domjudge Restart=always diff --git a/icpc-wf/ansible/files/judgedaemon1.service b/icpc-wf/ansible/files/judgedaemon1.service new file mode 100644 index 00000000..cb8f4b78 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon1.service @@ -0,0 +1,18 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target +Requires=create_cgroups.service +After=create_cgroups.service + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 1 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon2.service b/icpc-wf/ansible/files/judgedaemon2.service new file mode 100644 index 00000000..8582831c --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon2.service @@ -0,0 +1,18 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target +Requires=create_cgroups.service +After=create_cgroups.service + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 2 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon3.service b/icpc-wf/ansible/files/judgedaemon3.service new file mode 100644 index 00000000..72ee4637 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon3.service @@ -0,0 +1,18 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target +Requires=create_cgroups.service +After=create_cgroups.service + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 3 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon4.service b/icpc-wf/ansible/files/judgedaemon4.service new file mode 100644 index 00000000..4d667815 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon4.service @@ -0,0 +1,18 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target +Requires=create_cgroups.service +After=create_cgroups.service + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 4 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon5.service b/icpc-wf/ansible/files/judgedaemon5.service new file mode 100644 index 00000000..e2da6e91 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon5.service @@ -0,0 +1,18 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target +Requires=create_cgroups.service +After=create_cgroups.service + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 5 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/hosts b/icpc-wf/ansible/hosts index b41f495a..a4fde50f 100644 --- a/icpc-wf/ansible/hosts +++ b/icpc-wf/ansible/hosts @@ -1,15 +1,5 @@ [domserver] -domjudge-primary ansible_host=10.3.3.215 ansible_user=root -domjudge-backup ansible_host=10.3.3.216 ansible_user=root +domjudge-primary ansible_host=calca ansible_user=root [judgehost] -domjudge-judgehost1 ansible_host=10.2.2.192 ansible_user=root -domjudge-judgehost2 ansible_host=10.2.2.193 ansible_user=root -domjudge-judgehost3 ansible_host=10.2.2.194 ansible_user=root -domjudge-judgehost4 ansible_host=10.2.2.195 ansible_user=root -domjudge-judgehost5 ansible_host=10.2.2.196 ansible_user=root -domjudge-judgehost6 ansible_host=10.2.2.197 ansible_user=root -domjudge-judgehost7 ansible_host=10.2.2.198 ansible_user=root -domjudge-judgehost8 ansible_host=10.2.2.199 ansible_user=root -domjudge-judgehost9 ansible_host=10.2.2.200 ansible_user=root -domjudge-judgehost10 ansible_host=10.2.2.201 ansible_user=root +systest-judge ansible_host=systest-judge.domjudge.letstalk.nl ansible_user=root diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index bacf57fb..ebd00633 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -16,8 +16,6 @@ service: name=rsyslog enabled=yes state=restarted - name: restart systemctl shell: systemctl daemon-reload - - name: restart gdm - service: name=gdm3 enabled=yes state=restarted tasks: - name: include global variables include_vars: variables.yml @@ -29,6 +27,8 @@ - domjudge-run-1 - domjudge-run-2 - domjudge-run-3 + - domjudge-run-4 + - domjudge-run-5 - name: create domjudge-run group group: name=domjudge-run state=present @@ -56,15 +56,21 @@ - name: enable internal monitor file: path=/usr/share/X11/xorg.conf.d/22-icpc.conf state=absent + - name: remove apt-transport-https from bionic debootstrap file + lineinfile: + path: /usr/share/debootstrap/scripts/bionic + regexp: 'ca-certificates' + line: ' base="$base ca-certificates"' + - name: copy chroot DEB packages to install copy: src=files/install-chroot dest=/tmp/dj_ansible/ - name: create chroot shell: "{{DJ_DIR}}/misc-tools/dj_make_chroot -y -i openjdk-11-jdk-headless -l \"$(ls /tmp/dj_ansible/install-chroot/*.deb 2>/dev/null | tr '\n' ',')\" 2>&1 | tee /tmp/dj_make_chroot.log" environment: - DEBMIRROR: http://packages/ubuntu + DEBMIRROR: https://pc2cancer.ecs.csus.edu/ubuntu args: - creates: "/chroot/domjudge" + creates: "/chroot/domjudge-systest" - name: fix kernel parameters lineinfile: @@ -80,33 +86,35 @@ copy: src=files/{{item}}.service dest=/etc/systemd/system/ tags: updateservice with_items: - - create_cgroups - tune_cpu - - judgedaemon + - create_cgroups + - judgedaemon0 + - judgedaemon1 + - judgedaemon2 + - judgedaemon3 + - judgedaemon4 + - judgedaemon5 notify: restart systemctl - name: make sure systemctl is restarted meta: flush_handlers + - name: enable and restart the tune_cpu service + service: name=tune_cpu enabled=yes state=restarted + when: inventory_hostname == "porto-host" + - name: enable and restart the services we just copied service: name={{item}} enabled=yes state=restarted with_items: - create_cgroups - - tune_cpu - - judgedaemon - - - name: add autostart shortcuts - copy: src=files/{{item}}.desktop dest=/home/domjudge/.config/autostart/ owner=domjudge group=domjudge mode=0755 - with_items: - - taillog - - rotate - - domjudgelogo + - judgedaemon0 - - name: disable systemd timers - command: systemctl mask {{item}} - args: - creates: /etc/systemd/system/{{item}} + - name: enable and restart other judgehosts + service: name={{item}} enabled=yes state=restarted with_items: - - apt-daily-upgrade.timer - - apt-daily.timer - - systemd-tmpfiles-clean.timer + - judgedaemon1 + - judgedaemon2 + - judgedaemon3 + - judgedaemon4 + - judgedaemon5 + when: inventory_hostname != "porto-host" diff --git a/icpc-wf/ansible/variables.yml b/icpc-wf/ansible/variables.yml index fb539767..011eb915 100644 --- a/icpc-wf/ansible/variables.yml +++ b/icpc-wf/ansible/variables.yml @@ -1,21 +1,21 @@ # Directory of the domjudge repository checkout. -DJ_DIR: /home/domjudge/domjudge +DJ_DIR: /opt/domjudge-systest # Branch to checkout and use. -DJ_BRANCH: master +DJ_BRANCH: wf2019 # Set this to change the web interface background color. #BACKGROUND_COLOR: '#ddddff' # URL and IP of domserver from judgehosts. A hostname 'domserver' with # DOMSERVER_IP will be added to the judgehost /etc/hosts file. -DOMSERVER: https://domjudge -DOMSERVER_IP: 10.3.3.215 +DOMSERVER: https://systest.domjudge.org +DOMSERVER_IP: 131.155.69.89 DOMSERVER_URL: "{{DOMSERVER}}" -DOMSERVER_SSL_CERT: /etc/ssl/certs/domserver.crt -DOMSERVER_SSL_KEY: /etc/ssl/private/domserver.key +DOMSERVER_SSL_CERT: /etc/letsencrypt/live/systest.domjudge.org/fullchain.pem +DOMSERVER_SSL_KEY: /etc/letsencrypt/live/systest.domjudge.org/privkey.pem -TIMEZONE: "Europe/Lisbon" +TIMEZONE: "Europe/Amsterdam" PHP_FPM_MAX_CHILDREN: 300 PHP_MEMORY_LIMIT: 1024M @@ -40,11 +40,7 @@ API_PASSWORD: RESTAPI_PASSWORD #DJ_SHELL_USER_PW: some-hashed-password # Git repo URL -DJ_GIT_REPO: domjudge@10.3.3.223:domjudge - -# If using a Git repo which requires a SSH key, set the host and key here -DJ_GIT_HOST: 10.3.3.223 -DJ_GIT_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKayJQxbraoLvi7iYQ7TmTN08Spr1eFfcU/eqyfmacXDVSUMOn/LwwA0nh/bjkbtZSM6CFjQl2L9SXvlxSG9bYV2gYfOL4COxiVGPdVk783FcQlt3x+y4zFRZgz5FgJuGfRxWAqZstsjYceg1xQKkIFQLm+gup1EnLBcwKPARonDkRIa+5XYoKsGaRu1HFrzgGNIR1gmDXP1UAUgHz8MkELazNp1zTt7s7szFhNWhIdtBWbghrnRMss1W+qlx6umhd3T6y4EeJLxoUDYhbIQUCcBx+Rpf5sj/4LmgdCbHQS2OkXjaYtM4MGxEvfSrNT14rIV7HKrCr7BovVoj+p2El nicky@dyn070180.nbw.tue.nl +DJ_GIT_REPO: https://github.com/DOMjudge/domjudge # Key for domjudge user DJ_SSH_KEY: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEW4j7DOQ/BGT6ATtBLAcUHGVuHyydqs7E31DMbcX5uO icpc2018@domjudge-ccsadmin2