Merge branch 'develop' of github.com:centerforopenscience/osf.io into…

… feature/ember-osf-web

.docker-compose.sharejs.env

@@ -1,7 +1,7 @@
 SHAREJS_SERVER_HOST=0.0.0.0
 SHAREJS_SERVER_PORT=7007
 SHAREJS_DB_NAME=sharejs
-SHAREJS_DB_URL=mongodb://192.168.168.167:27017/sharejs
+SHAREJS_DB_URL=mongodb://192.168.168.167:27017/sharejs?ssl=true
 
 #SHAREJS_CORS_ALLOW_ORIGIN=
 #SHAREJS_SENTRY_DSN=

.dockerignore

@@ -7,3 +7,4 @@ node_modules
 *.pyc
 **/*.pyc
 docker*
+ssl/

.gitignore

@@ -194,3 +194,4 @@ scripts/image_maniplation/test_rounded_corners.html
 docker-compose.override.yml
 .unison*
 .docker-sync/
+ssl/

Dockerfile

@@ -6,14 +6,14 @@ ENV GOSU_VERSION=1.10 \
     YARN_VERSION=1.1.0
 
 # Libraries such as matplotlib require a HOME directory for cache and configuration
-RUN usermod -d /home www-data \
-    && chown www-data:www-data /home \
-    # https://github.com/nodejs/docker-node/blob/9c25cbe93f9108fd1e506d14228afe4a3d04108f/8.2/Dockerfile
-    # gpg keys listed at https://github.com/nodejs/node#release-team
-    && set -ex \
+RUN set -ex \
+    && mkdir -p /var/www \
+    && chown www-data:www-data /var/www \
     # GOSU
     && gpg --keyserver pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
     && for key in \
+      # https://github.com/nodejs/docker-node/blob/9c25cbe93f9108fd1e506d14228afe4a3d04108f/8.2/Dockerfile
+      # gpg keys listed at https://github.com/nodejs/node#release-team
       # Node
       9554F04D7259F04124DE6B476D5A82AC7E37093B \
       94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \

addons/wiki/settings/defaults.py

@@ -1,5 +1,5 @@
 import datetime
-
+import os
 import pytz
 
 from website import settings
@@ -9,7 +9,7 @@
 SHAREJS_URL = '{}:{}'.format(SHAREJS_HOST, SHAREJS_PORT)
 
 SHAREJS_DB_NAME = 'sharejs'
-SHAREJS_DB_URL = 'mongodb://{}:{}/{}'.format(settings.DB_HOST, settings.DB_PORT, SHAREJS_DB_NAME)
+SHAREJS_DB_URL = os.environ.get('SHAREJS_DB_URL', 'mongodb://{}:{}/{}'.format(settings.DB_HOST, settings.DB_PORT, SHAREJS_DB_NAME))
 
 # TODO: Change to release date for wiki change
 WIKI_CHANGE_DATE = datetime.datetime.utcfromtimestamp(1423760098).replace(tzinfo=pytz.utc)

addons/wiki/utils.py

@@ -3,6 +3,7 @@
 import urllib
 import uuid
 
+import ssl
 from pymongo import MongoClient
 import requests
 
@@ -101,7 +102,7 @@ def migrate_uuid(node, wname):
 
 def share_db():
     """Generate db client for sharejs db"""
-    client = MongoClient(wiki_settings.SHAREJS_DB_URL)
+    client = MongoClient(wiki_settings.SHAREJS_DB_URL, ssl_cert_reqs=ssl.CERT_NONE)
     return client[wiki_settings.SHAREJS_DB_NAME]
 
 

api/base/views.py

@@ -145,7 +145,7 @@ def get_serializer_context(self):
         else:
             embeds = self.request.query_params.getlist('embed') or self.request.query_params.getlist('embed[]')
 
-        fields_check = self.serializer_class._declared_fields.copy()
+        fields_check = self.get_serializer_class()._declared_fields.copy()
         if 'fields[{}]'.format(self.serializer_class.Meta.type_) in self.request.query_params:
             # Check only requested and mandatory fields
             sparse_fields = self.request.query_params['fields[{}]'.format(self.serializer_class.Meta.type_)]

api/nodes/serializers.py

@@ -815,6 +815,7 @@ class NodeContributorsCreateSerializer(NodeContributorsSerializer):
     users = RelationshipField(
         related_view='users:user-detail',
         related_view_kwargs={'user_id': '<user._id>'},
+        always_embed=True,
         required=False
     )
 

api_tests/files/views/test_file_detail.py

@@ -10,7 +10,7 @@
 from api.base.settings.defaults import API_BASE
 from api_tests import utils as api_utils
 from framework.auth.core import Auth
-from osf.models import NodeLog, Session
+from osf.models import NodeLog, Session, QuickFilesNode
 from osf_tests.factories import (
     AuthUserFactory,
     CommentFactory,
@@ -463,6 +463,19 @@ def test_folder_files_relationships_contains_guid_not_id(self, app, user, node):
         assert node._id in split_href
         assert node.id not in split_href
 
+    def test_embed_user_on_quickfiles_detail(self, app, user):
+        quickfiles = QuickFilesNode.objects.get(creator=user)
+        osfstorage = quickfiles.get_addon('osfstorage')
+        root = osfstorage.get_root()
+        test_file = root.append_file('speedyfile.txt')
+
+        url = '/{}files/{}/?embed=user'.format(API_BASE, test_file._id)
+        res = app.get(url, auth=user.auth)
+
+        assert res.json['data'].get('embeds', None)
+        assert res.json['data']['embeds'].get('user')
+        assert res.json['data']['embeds']['user']['data']['id'] == user._id
+
 
 @pytest.mark.django_db
 class TestFileVersionView:

docker-compose.override.yml

@@ -105,6 +105,23 @@ services:
 #    volumes:
 #      - emberosf-sync:/code:nocopy
 
+#  ####################
+#  # RabbitMQ SSL
+#  # Enable this, place the certs in ./ssl, and uncomment the BROKER_USE_SSL dictionary in local.py
+#  # Uncomment lines under worker in docker-compose
+#  #####################
+#  rabbitmq:
+#    ports:
+#      - 5671:5671
+#    environment:
+#      RABBITMQ_SSL_CERTFILE: /etc/ssl/server_certificate.pem
+#      RABBITMQ_SSL_KEYFILE: /etc/ssl/server_key.pem
+#      RABBITMQ_SSL_CACERTFILE: /etc/ssl/ca_certificate.pem
+#    volumes:
+#      - ./ssl/celery-server.cert.pem:/etc/ssl/server_certificate.pem:ro
+#      - ./ssl/celery-server.key.pem:/etc/ssl/server_key.pem:ro
+#      - ./ssl/ca-chain.cert.pem:/etc/ssl/ca_certificate.pem:ro
+
 volumes:
   osf-sync:
     external: true

docker-compose.yml

@@ -5,7 +5,7 @@ version: '3'
 volumes:
   postgres_data_vol:
     external: false
-  tokumx_data_vol:
+  mongo_data_vol:
     external: false
   elasticsearch_data_vol:
     external: false
@@ -59,12 +59,12 @@ services:
 
   postgres:
     image: postgres:9.6
-    command: |-
-      /bin/bash -c "
-        echo \"$$POSTGRES_INITDB\" > /docker-entrypoint-initdb.d/commands.sh &&
+    command:
+      - /bin/bash
+      - -c
+      - echo "$$POSTGRES_INITDB" > /docker-entrypoint-initdb.d/commands.sh &&
         chmod +x /docker-entrypoint-initdb.d/commands.sh &&
         /docker-entrypoint.sh postgres
-      "
     ports:
       - 5432:5432
     environment:
@@ -76,15 +76,22 @@ services:
       - "${POSTGRES_DATA_VOL:-postgres_data_vol}:/var/lib/postgresql/data/"
     stdin_open: true
 
-  tokumx:
-    image: quay.io/centerforopenscience/tokumx:latest
-    command: mongod --ipv6
+  mongo:
+    image: mongo:3.4
+    command: 
+      - /bin/bash
+      - -c 
+      - openssl req -newkey rsa:2048 -new -x509 -days 9999 -nodes 
+          -subj "/C=US/ST=Virginia/L=Charlottesville/O=COS/OU=Test/CN=192.168.168.167"
+          -out /etc/ssl/mongodb.crt -keyout /etc/ssl/mongodb.key &&
+        cat /etc/ssl/mongodb.key /etc/ssl/mongodb.crt > /etc/ssl/mongodb.pem &&
+        chown -R mongodb:mongodb /etc/ssl /data/db &&
+        chmod -R 0600 /etc/ssl/* &&
+        gosu mongodb mongod --sslMode requireSSL --sslDisabledProtocols=TLS1_0,TLS1_1 --sslPEMKeyFile /etc/ssl/mongodb.pem
     ports:
       - 27017:27017
-    environment:
-      TOKU_HUGE_PAGES_OK: 1
     volumes:
-      - tokumx_data_vol:/data/db
+      - mongo_data_vol:/data/db
     stdin_open: true
 
   rabbitmq:
@@ -113,13 +120,13 @@ services:
 
   mfr_requirements:
     image: quay.io/centerforopenscience/mfr:develop
-    command: |-
-      /bin/bash -c "
-        invoke install --develop &&
+    command:
+      - /bin/bash
+      - -c
+      - invoke install --develop &&
         (python -m compileall /usr/local/lib/python3.5 || true) &&
         rm -Rf /python3.5/* &&
         cp -Rf -p /usr/local/lib/python3.5 /
-      "
     restart: 'no'
     volumes:
       - mfr_requirements_vol:/python3.5
@@ -144,13 +151,13 @@ services:
 
   wb_requirements:
     image: quay.io/centerforopenscience/waterbutler:develop
-    command: |-
-      /bin/bash -c "
-        invoke install --develop &&
+    command:
+      - /bin/bash
+      - -c
+      - invoke install --develop &&
         (python -m compileall /usr/local/lib/python3.5 || true) &&
         rm -Rf /python3.5/* &&
         cp -Rf -p /usr/local/lib/python3.5 /
-      "
     restart: 'no'
     volumes:
       - wb_requirements_vol:/python3.5
@@ -230,7 +237,12 @@ services:
 
   preprints:
     image: quay.io/centerforopenscience/ember-preprints:develop-development-local
-    command: /bin/bash -c "yarn --pure-lockfile --ignore-engines --no-progress --no-emoji && ./node_modules/bower/bin/bower install --allow-root --config.interactive=false && ./node_modules/ember-cli/bin/ember serve --host 0.0.0.0 --port 4200 --live-reload-port 41953"
+    command:
+      - /bin/bash
+      - -c 
+      - yarn --pure-lockfile --ignore-engines --no-progress --no-emoji &&
+       ./node_modules/.bin/bower install --allow-root --config.interactive=false &&
+       ./node_modules/.bin/ember serve --host 0.0.0.0 --port 4200 --live-reload-port 41953
     restart: unless-stopped
     depends_on:
       - api
@@ -253,7 +265,12 @@ services:
 
   registries:
     image: quay.io/centerforopenscience/ember-osf-registries:develop-development-local
-    command: /bin/bash -c "yarn --pure-lockfile --ignore-engines --no-progress --no-emoji && ./node_modules/bower/bin/bower install --allow-root --config.interactive=false && ./node_modules/ember-cli/bin/ember serve --host 0.0.0.0 --port 4300"
+    command: 
+      - /bin/bash
+      - -c
+      - yarn --pure-lockfile --ignore-engines --no-progress --no-emoji &&
+       ./node_modules/.bin/bower install --allow-root --config.interactive=false &&
+       ./node_modules/.bin/ember serve --host 0.0.0.0 --port 4300
     restart: unless-stopped
     depends_on:
       - api
@@ -273,8 +290,12 @@ services:
   ##############
 
   reviews:
-    image: quay.io/centerforopenscience/ember-osf-reviews:develop-development-local
-    command: /bin/bash -c "yarn --pure-lockfile --no-progress --no-emoji && ./node_modules/ember-cli/bin/ember serve --host 0.0.0.0 --port 4400"
+    image: quay.io/centerforopenscience/osf-reviews:develop
+    command:
+      - /bin/bash
+      - -c 
+      - yarn --pure-lockfile --no-progress --no-emoji &&
+        ./node_modules/.bin/ember serve --host 0.0.0.0 --port 4400
     restart: unless-stopped
     depends_on:
       - api
@@ -297,13 +318,13 @@ services:
 
   requirements:
     image: quay.io/centerforopenscience/osf:develop
-    command: |-
-      /bin/bash -c "
-        invoke requirements --quick &&
+    command:
+      - /bin/bash
+      - -c
+      - invoke requirements --quick &&
         (python -m compileall /usr/local/lib/python2.7 || true) &&
         rm -Rf /python2.7/* &&
         cp -Rf -p /usr/local/lib/python2.7 /
-      "
     restart: 'no'
     environment:
       DJANGO_SETTINGS_MODULE: api.base.settings
@@ -342,7 +363,8 @@ services:
     ports:
       - 7007:7007
     depends_on:
-      - tokumx
+      - mongo
+      - web
     env_file:
       - .docker-compose.sharejs.env
     volumes:
@@ -377,12 +399,16 @@ services:
     environment:
       C_FORCE_ROOT: 1
       DJANGO_SETTINGS_MODULE: api.base.settings
+#      BROKER_URL: amqp://guest:[email protected]:5671/
     env_file:
       - .docker-compose.env
     volumes:
       - osf_requirements_vol:/usr/local/lib/python2.7
       - osf_bower_components_vol:/code/website/static/vendor/bower_components
       - osf_node_modules_vol:/code/node_modules
+#      - ./ssl/ca-chain.cert.pem:/etc/ssl/certs/ca-chain.cert.pem:ro
+#      - ./ssl/celery-client.cert.pem:/etc/ssl/certs/worker.pem:ro
+#      - ./ssl/celery-client.key.pem:/etc/ssl/private/worker.key:ro
     stdin_open: true
 
   admin:
@@ -441,6 +467,7 @@ services:
       DJANGO_SETTINGS_MODULE: api.base.settings
     env_file:
       - .docker-compose.env
+      - .docker-compose.sharejs.env
     volumes:
       - osf_requirements_vol:/usr/local/lib/python2.7
       - osf_bower_components_vol:/code/website/static/vendor/bower_components

framework/celery_tasks/__init__.py

@@ -17,6 +17,8 @@
     client = Client(settings.SENTRY_DSN, release=settings.VERSION, tags={'App': 'celery'})
     register_signal(client)
 
+if settings.BROKER_USE_SSL:
+    app.setup_security()
 
 @app.task
 def error_handler(task_id, task_name):

karma.common.conf.js

@@ -32,6 +32,9 @@ var webpackTestConfig = {
             // Assume test files are ES6
             {test: /\.test\.js$/, loader: 'babel-loader'},
         ])
+    },
+        node: {
+       fs: 'empty'
     }
 };
 

package.json

@@ -44,6 +44,7 @@
     "lodash.includes": "^4.3.0",
     "lodash.set": "^4.3.2",
     "markdown-it": "~4.1.0",
+    "markdown-it-imsize": "^2.0.1",
     "markdown-it-ins-del": "^0.1.1",
     "markdown-it-sanitizer": "~0.3.0",
     "@centerforopenscience/markdown-it-toc": "~1.1.1",

webpack.common.config.js

@@ -222,5 +222,8 @@ module.exports = {
             //Dirty hack because mime-type's json file is "special"
             {test: /db.json/, loader: 'json-loader'}
         ]
+    },
+    node: {
+       fs: 'empty'
     }
 };

website/settings/defaults.py

@@ -414,6 +414,7 @@ def parent_dir(path):
 RABBITMQ_VHOST = os.environ.get('RABBITMQ_VHOST', '/')
 
 BROKER_URL = os.environ.get('BROKER_URL', 'amqp://{}:{}@{}:{}/{}'.format(RABBITMQ_USERNAME, RABBITMQ_PASSWORD, RABBITMQ_HOST, RABBITMQ_PORT, RABBITMQ_VHOST))
+BROKER_USE_SSL = False
 
 # Default RabbitMQ backend
 CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', BROKER_URL)

website/settings/local-dist.py

@@ -85,6 +85,16 @@
 ## Default RabbitMQ broker
 BROKER_URL = 'amqp://'
 
+# Celery with SSL
+# import ssl
+#
+# BROKER_USE_SSL = {
+#     'keyfile': '/etc/ssl/private/worker.key',
+#     'certfile': '/etc/ssl/certs/worker.pem',
+#     'ca_certs': '/etc/ssl/certs/ca-chain.cert.pem',
+#     'cert_reqs': ssl.CERT_REQUIRED,
+# }
+
 # Default RabbitMQ backend
 CELERY_RESULT_BACKEND = 'amqp://'