Merge branch 'feature/crafternode' into develop

chhofi · Aug 3, 2015 · 2258b25 · 2258b25
2 parents 860bce1 + 5c2a002
commit 2258b25
Show file tree

Hide file tree

Showing 55 changed files with 1,257 additions and 668 deletions.
diff --git a/README.rst b/README.rst
@@ -6,12 +6,14 @@ ansible-raycrafter
     :target: http://ansible-raycrafter.readthedocs.org/en/latest/
     :alt: Documentation
 
-Ansible_ Playbook designed for running a Raycrafter_ Master Server.
+Ansible_ Playbook designed for running a Raycrafter_ Master Server and Crafter Nodes.
 It installs and configures applications that are needed for production deployments.
 
 It deploys a Django_ project and sets up Gunicorn_ and Nginx_ to serve your site.
 PostgreSQL_ is used as database backend for Django_.
-Celery_ (with RabbitMQ_ as message broker) for asynchronous task queue/job queue.
+RabbitMQ_ for sending task to Celery_ workers (a asynchronous task queue).
+The workers live on the crafter nodes. They can also transfer files via GridFTP_
+to the cluster.
 
 On top of that a logging server is deployed. In this case it is Graylog_, which depends
 on Elasticsearch_ and MongoDB_.
@@ -29,6 +31,7 @@ Overview:
 - Elasticsearch_
 - MongoDB_
 - Graylog_ Server/Web Interface
+- GridFTP_
 
 **Tested with OS:** Ubuntu 14.04 LTS x64
 
@@ -110,3 +113,4 @@ Useful Links
 .. _Djangotest: https://github.com/RayCrafter/djangotest
 .. _Cookiecutter: https://github.com/audreyr/cookiecutter
 .. _Cookiecutter-Django: https://github.com/RayCrafter/cookiecutter-django
+.. _GridFTP: http://toolkit.globus.org/toolkit/docs/latest-stable/gridftp/
diff --git a/Vagrantfile b/Vagrantfile
@@ -4,24 +4,40 @@
 # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 VAGRANTFILE_API_VERSION = "2"
 
+groups = {
+  "masterservers" => ['master'],
+  "crafternodes" => ['crafter'],
+}
+
+inventory = {
+  "master" => {:ip => "192.168.33.15", :cpus => 2, :mem => 4096},
+  "crafter" => {:ip => "192.168.33.16", :cpus => 1, :mem => 1024},
+}
+
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
 
-  config.vm.network :private_network, ip: "192.168.33.15"
 
-  config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--name", "djangotest", "--memory", "4096", "--cpus", "2"]
-  end
+  inventory.each_with_index do |(hostname, info), index|
+    config.vm.define hostname do |cfg|
 
-  # Shared folder from the host machine to the guest machine. Uncomment the line
-  # below to enable it.
-  #config.vm.synced_folder "../../../my-cool-app", "/webapps/mycoolapp/my-cool-app"
+      cfg.vm.provider :virtualbox do |vb, override|
+        override.vm.box = "ubuntu/trusty64"
+        override.vm.network :private_network, ip: "#{info[:ip]}"
+        override.vm.hostname = hostname
+        vb.name = 'raycrafter-' + hostname
+        vb.customize ["modifyvm", :id, "--memory", info[:mem], "--cpus", info[:cpus]]
+      end
 
-  # Ansible provisioner.
-  config.vm.provision "ansible" do |ansible|
-    ansible.playbook = "vagrant.yml"
-    ansible.host_key_checking = false
-    ansible.verbose = "v"
-    ansible.ask_vault_pass = true
+      # provision nodes with ansible
+      if index == inventory.size - 1
+        cfg.vm.provision :ansible do |ansible|
+          ansible.groups = groups
+          ansible.verbose = "v"
+          ansible.playbook = "site.yml"
+          ansible.limit = 'all'# "#{info[:ip]}" # Ansible hosts are identified by ip
+          ansible.vault_password_file = "vaultpwfile.txt"
+        end
+      end
+    end
   end
 end
diff --git a/base.yml b/base.yml
@@ -0,0 +1,12 @@
+---
+
+- name: Common base role
+  hosts: [masterservers, crafternodes]
+  sudo: yes
+  sudo_user: root
+  remote_user: vagrant
+  vars_files:
+    - env_vars/secret.yml
+    - env_vars/base.yml
+  roles:
+    - base
diff --git a/crafternodes.yml b/crafternodes.yml
@@ -0,0 +1,29 @@
+---
+
+# Add masterservers to hosts but disable all roles
+# This is necessary for gathering facts
+# So we can set the right hostname for rabbitmq and the db
+- name: Create RayCrafter crafter node
+  hosts: [crafternodes, masterservers]
+  become: yes
+  become_user: root
+  remote_user: vagrant
+  vars_files:
+    - env_vars/secret.yml
+    - env_vars/base.yml
+    - env_vars/crafternode.yml
+  roles:
+    - {role: supervisor, when: "'crafternodes' in group_names"}
+    - {role: web, when: "'crafternodes' in group_names"}
+    - {role: celery, when: "'crafternodes' in group_names"}
+    - {role: gridftp, when: "'crafternodes' in group_names"}
+  tasks:
+    - name: Create .ssh key dir
+      become_user: "{{ web_user}}"
+      file: path="~/.ssh" state=directory
+      when: "'crafternodes' in group_names"
+    - name: Copy private ssh key of crafternodes
+      become_user: "{{ web_user }}"
+      copy: src="{{playbook_dir}}/files/ssh/id_rsa" dest="~/.ssh/id_rsa"
+            owner={{ web_user }} mode=0700
+      when: "'crafternodes' in group_names"
diff --git a/docs/source/ansibleguide.rst b/docs/source/ansibleguide.rst
@@ -6,12 +6,15 @@ Ansible Guide
 
 Before you start, see the configuration section. There are some thing you need to setup first.
 
-First, create an inventory file for the environment. There are two categories: ``masterservers`` and ``hlrs``. The first category will deploy a RayCrafter Master Server. The second will setup everything at the cluster.
+First, create an inventory file for the environment. There are three categories: ``masterservers``, ``crafternodes`` and ``hlrs``. The first category will deploy a RayCrafter Master Server. The second will setup a note that has ssh access to the cluster and submits jobs or transfers data. The third will setup everything at the cluster.
 
 For example create a file ``hosts`` with the content::
 
   # content of hosts inventory file
   [masterservers]
+  141.62.110.219 ansible_ssh_user=useronserver
+
+  [crafternodes]
   141.62.110.220 ansible_ssh_user=useronserver
   
   [hlrs]
@@ -24,10 +27,3 @@ Run the main playbook ``sites.yml``, which includes the production and hlrs play
 If you're testing with vagrant, and the VM is already running, you can use this command::
 
   $ vagrant provision
-
-or alternatively::
-
-  $ ansible-playbook -i vagrant_ansible_inventory_default --private-key=~/.vagrant.d/insecure_private_key vagrant.yml  --ask-vault-pass
-
-
-
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
@@ -14,7 +14,7 @@ In the ansible inventory file you can configure which machines to provision. See
 Playbook
 --------
 
-The next big step is your playbook. Here you can configure which roles to use on your hosts and what variables to use. Have a look at ``production.yml`` or ``vagrant.yml``.
+The next big step is your playbook. Here you can configure which roles to use on your hosts and what variables to use. Have a look at ``site.yml``.
 
 ---------
 Variables
@@ -49,28 +49,16 @@ The contents of ``env_vars/secret.yml`` should be something like but with actual
 Files
 -----
 
-In order to use GridFTP, you need to have a X.509 certificate and key. See the `HLRS GridFTP Wiki <https://wickie.hlrs.de/platforms/index.php/Data_Transfer_with_GridFTP>`_. By default, you have to put them in ``/files/ssl/usercert.pem`` and ``/files/ssl/userkey.pem``.
-See :ref:`raycrafterdoc:gridftp`.
-
-There are some files that you might want to replace/encrypt, such as ssl/ssh keys.
-For ssl certificates, create/override ``/files/ssl/application.pem``, ``/files/ssl/application.key_unencrypted``. The key should be encryptes::
+There are some files that you have to create such as ssh/ssl keys.
 
-  $ openssl aes-256-cbc -salt -a -e -in files/ssl/application.key_unencrypted -out files/ssl/application.key -k "YourSSLKeyPassword"
+For ssl certificates, create/override ``/files/ssl/application.pem``, ``/files/ssl/application.key`` with your certificate and private key.
 
-Do not commit the unencrypted version of the key!
+In order to use GridFTP, you need to have a X.509 certificate and key. See the `HLRS GridFTP Wiki <https://wickie.hlrs.de/platforms/index.php/Data_Transfer_with_GridFTP>`_. By default, you have to put them in ``/files/ssl/usercert.pem`` and ``/files/ssl/userkey.pem``.
+See :ref:`raycrafterdoc:gridftp`.
 
 You should also create an ssh-key for the master server so he can access the cluster via ssh.
 Create a ssh keypair with::
 
   $ ssh-keygen -t rsa -b 4096 -C "raycrafter master server"
 
-Move them to ``/files/ssh/id_rsa_unencrypted`` and ``/files/ssh/id_rsa.pub``.
-Encrypt the key::
-
-    $ openssl aes-256-cbc -salt -a -e -in files/ssh/id_rsa_unencrypted -out files/ssh/id_rsa -k "YourSSHKeyPassword"
-
-Do not commit the unencrypted version of the key!
-
-Store the password in ``/env_vars/secret.yml`` as ``ssl_key_password`` and ``ssh_key_password``. Make sure you envrypt that file with ansible vault::
-
-  $ ansible-vault encrypt ./env_vars/secret.yml
+Move them to ``/files/ssh/id_rsa`` and ``/files/ssh/id_rsa.pub``.
diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
@@ -24,7 +24,7 @@ Install all ansible requirements. You might have to specify a path for the roles
 Configuration
 +++++++++++++
 
-There are two sets of configurations: ``env_vars/vagrant.yml`` and ``env_vars/production.yml``.
+The main configuration files are in ``env_vars/``.
 Here you configure your setup, like the location of your Git_ project, the project name, and application name which will be used throughout the Ansible_ configuration.
 For more information see :ref:`configuration`.
 
@@ -52,7 +52,7 @@ After installing Ansible_, Vagrant_ and VirtualBox_ you simply execute::
           the encrypted shh and ssl keys. If you do not have these passwords, create your
           own keys. See the configuration section.
 
-You have to enter the ansible vault password, once provisioning starts!
+You have to create a ``vaultpwfile.txt`` with the vault password in it.
 Wait a few minutes for the magic to happen. Access the Django_ site by goingto this URL: https://192.168.33.15
 
 Access the Graylog_ web-interface via: https://192.168.33.15:9000

diff --git a/docs/source/roles/celery.rst b/docs/source/roles/celery.rst
@@ -7,7 +7,7 @@ Dependencies:
 - web
 - supervisor
 
-Copies the startup and supervisor scripts. Will start workers and celerycam.
+Copies the startup and supervisor scripts. Will start workers.
 Celerycam is a monitoring service for celery workers and tasks.
 Create log dirs and files.
 Notifies supervisor to restart the celery app.
@@ -19,18 +19,15 @@ Variables
 ======================== ======================================================= ==================================================
 Name                     Default                                                 Description
 ======================== ======================================================= ==================================================
-celery_user              {{ gunicorn_user }}                                     The user which executes celery workers
-celery_group             {{ gunicorn_group }}                                    The usergroup
+celery_user              {{ web_user }}                                          The user which executes celery workers
+celery_group             {{ web_group }}                                         The usergroup
 celery_application_name  celery                                                  The name off the celery programm
 celery_scripts_dir       {{ virtualenv_path }}/scripts/celery                    Path to the celery executable
 celery_template_file     {{ celery_application_name }}_start.j2                  The template to use for the start script
-celerycam_template_file: "celerycam_start.j2"                                    The template to use for celerycam. A monitor
-                                                                                 service.
 celery_worker_app        {{ application_name }}                                  The celery app name (used in the -A flag of the
                                                                                  celery executable)
 celery_log_dir           {{ virtualenv_path }}/logs                              Dir for the log files
 celery_log_file          {{ celery_log_dir }}/{{ celery_application_name }}.log  Full path to the log file
-celerycam_log_file       "{{ celery_log_dir }}/celerycam.log"                    Full path to the celery cam log file
 celery_num_workers       2                                                       Starts this many workers
 ======================== ======================================================= ==================================================
 

diff --git a/docs/source/roles/celerycam.rst b/docs/source/roles/celerycam.rst
@@ -0,0 +1,41 @@
+=========
+Celerycam
+=========
+
+Dependencies:
+
+- web
+- supervisor
+
+Copies the startup and supervisor scripts. Will start the celerycam service.
+Celerycam is a monitoring service for celery workers and tasks.
+Create log dirs and files.
+Notifies supervisor to restart the celerycam service.
+
+---------
+Variables
+---------
+
+========================== ======================================================= ==================================================
+Name                       Default                                                 Description
+========================== ======================================================= ==================================================
+celerycam_user             {{ web_user }}                                          The user which executes celery workers
+celerycam_group            {{ web_group }}                                         The usergroup
+celerycam_scripts_dir      {{ virtualenv_path }}/scripts/celerycam                 Path to the celery executable
+celerycam_template_file    "celerycam_start.j2"                                    The template to use for celerycam. A monitor
+                                                                                   service.
+celerycam_log_dir          {{ virtualenv_path }}/logs                              Dir for the log files
+celerycam_log_file         "{{ celery_log_dir }}/celerycam.log"                    Full path to the celery cam log file
+========================== ======================================================= ==================================================
+
+----
+Tags
+----
+===================== ==========================================================================
+Name                  Description
+===================== ==========================================================================
+celerycam             Tag for all tasks
+supervisor            All supervisor related tasks, like copying the config or
+                      restarting the app
+===================== ==========================================================================
+
diff --git a/docs/source/roles/gridftp.rst b/docs/source/roles/gridftp.rst
@@ -26,6 +26,7 @@ gridftp_repo_creates_file "globus-toolkit-6-stable-trusty.list"
 ----
 Tags
 ----
+
 ===================== ==========================================================================
 Name                  Description
 ===================== ==========================================================================

diff --git a/docs/source/roles/gunicorn.rst b/docs/source/roles/gunicorn.rst
@@ -0,0 +1,50 @@
+========
+Gunicorn
+========
+
+Dependencies:
+
+- supervisor
+
+Create user and usergroup for gunicorn.
+Create a virtualenv for gunicorn.
+Install gunicorn.
+
+Create gunicorn start script and logging files.
+Create supervisor config files.
+Make the gunicorn user owner of the virtual env path.
+
+---------
+Variables
+---------
+
+========================== ======================================================================================================================== ==================================================
+Name                       Default                                                                                                                  Description
+========================== ======================================================================================================================== ==================================================
+application_name           "gunicorn"                                                                                                               Name of the application that gunicorn serves
+virtualenv_root            "/webapps"                                                                                                               Location for where to create virtual env
+virtualenv_path            "{{ virtualenv_root }}/{{ application_name }}"                                                                           Path to the virtual env
+application_log_dir        "{{ virtualenv_path }}/logs"                                                                                             Path to the log dir
+application_log_file       "{{ application_log_dir }}/gunicorn_supervisor.log"                                                                      Path to the log file
+gunicorn_user              "{{ application_name }}"                                                                                                 Username for the user running gunicorn
+gunicorn_group             webapps                                                                                                                  Groupname for the gunicorn user
+gunicorn_num_workers       3                                                                                                                        Numer of gunicorn workers
+gunicorn_max_requests      0                                                                                                                        Maximum requests before gunicorn restarts. 0 for
+                                                                                                                                                    no limit.
+gunicorn_timeout_seconds   30                                                                                                                       Maximum timeout for requests.
+gunicorn_workdir           /                                                                                                                        Workdir for executing gunicorn
+gunicorn_wsgi_file         ~                                                                                                                        Path to the wsgi file. Has to be importable.
+========================== ======================================================================================================================== ==================================================
+
+----
+Tags
+----
+
+===================== ==========================================================================
+Name                  Description
+===================== ==========================================================================
+virtualenv            Install packages
+supervisor            Create configs and restart supervisor
+gunicorn              Tag for all tasks
+===================== ==========================================================================
+
-Original file line number
+Diff line change
@@ Expand Up @@
     ----
     Tags
     ----
     ===================== ==========================================================================
     Name                  Description
     ===================== ==========================================================================
@@ Expand Down @@