diff --git a/.docker-compose.sharejs.env b/.docker-compose.sharejs.env index cbfa27f62d2..5d16845a8fb 100644 --- a/.docker-compose.sharejs.env +++ b/.docker-compose.sharejs.env @@ -1,7 +1,7 @@ SHAREJS_SERVER_HOST=0.0.0.0 SHAREJS_SERVER_PORT=7007 SHAREJS_DB_NAME=sharejs -SHAREJS_DB_URL=mongodb://192.168.168.167:27017/sharejs +SHAREJS_DB_URL=mongodb://192.168.168.167:27017/sharejs?ssl=true #SHAREJS_CORS_ALLOW_ORIGIN= #SHAREJS_SENTRY_DSN= diff --git a/.dockerignore b/.dockerignore index a25fac9aca5..d3711e80f6f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ node_modules *.pyc **/*.pyc docker* +ssl/ diff --git a/.gitignore b/.gitignore index 5b2dcd5a08b..be81e641324 100644 --- a/.gitignore +++ b/.gitignore @@ -194,3 +194,4 @@ scripts/image_maniplation/test_rounded_corners.html docker-compose.override.yml .unison* .docker-sync/ +ssl/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e225a6b1ee2..3e775b2a237 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,14 +6,14 @@ ENV GOSU_VERSION=1.10 \ YARN_VERSION=1.1.0 # Libraries such as matplotlib require a HOME directory for cache and configuration -RUN usermod -d /home www-data \ - && chown www-data:www-data /home \ - # https://github.com/nodejs/docker-node/blob/9c25cbe93f9108fd1e506d14228afe4a3d04108f/8.2/Dockerfile - # gpg keys listed at https://github.com/nodejs/node#release-team - && set -ex \ +RUN set -ex \ + && mkdir -p /var/www \ + && chown www-data:www-data /var/www \ # GOSU && gpg --keyserver pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ && for key in \ + # https://github.com/nodejs/docker-node/blob/9c25cbe93f9108fd1e506d14228afe4a3d04108f/8.2/Dockerfile + # gpg keys listed at https://github.com/nodejs/node#release-team # Node 9554F04D7259F04124DE6B476D5A82AC7E37093B \ 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ diff --git a/addons/wiki/settings/defaults.py b/addons/wiki/settings/defaults.py index 325e5a1f1c9..5b3f5b33235 100644 --- a/addons/wiki/settings/defaults.py +++ b/addons/wiki/settings/defaults.py @@ -1,5 +1,5 @@ import datetime - +import os import pytz from website import settings @@ -9,7 +9,7 @@ SHAREJS_URL = '{}:{}'.format(SHAREJS_HOST, SHAREJS_PORT) SHAREJS_DB_NAME = 'sharejs' -SHAREJS_DB_URL = 'mongodb://{}:{}/{}'.format(settings.DB_HOST, settings.DB_PORT, SHAREJS_DB_NAME) +SHAREJS_DB_URL = os.environ.get('SHAREJS_DB_URL', 'mongodb://{}:{}/{}'.format(settings.DB_HOST, settings.DB_PORT, SHAREJS_DB_NAME)) # TODO: Change to release date for wiki change WIKI_CHANGE_DATE = datetime.datetime.utcfromtimestamp(1423760098).replace(tzinfo=pytz.utc) diff --git a/addons/wiki/utils.py b/addons/wiki/utils.py index ce96b35acf9..b387fa4ee73 100644 --- a/addons/wiki/utils.py +++ b/addons/wiki/utils.py @@ -3,6 +3,7 @@ import urllib import uuid +import ssl from pymongo import MongoClient import requests @@ -101,7 +102,7 @@ def migrate_uuid(node, wname): def share_db(): """Generate db client for sharejs db""" - client = MongoClient(wiki_settings.SHAREJS_DB_URL) + client = MongoClient(wiki_settings.SHAREJS_DB_URL, ssl_cert_reqs=ssl.CERT_NONE) return client[wiki_settings.SHAREJS_DB_NAME] diff --git a/api/base/views.py b/api/base/views.py index 21026757065..b8e49854073 100644 --- a/api/base/views.py +++ b/api/base/views.py @@ -145,7 +145,7 @@ def get_serializer_context(self): else: embeds = self.request.query_params.getlist('embed') or self.request.query_params.getlist('embed[]') - fields_check = self.serializer_class._declared_fields.copy() + fields_check = self.get_serializer_class()._declared_fields.copy() if 'fields[{}]'.format(self.serializer_class.Meta.type_) in self.request.query_params: # Check only requested and mandatory fields sparse_fields = self.request.query_params['fields[{}]'.format(self.serializer_class.Meta.type_)] diff --git a/api/nodes/serializers.py b/api/nodes/serializers.py index 0eecfa4995e..01e6ac0efee 100644 --- a/api/nodes/serializers.py +++ b/api/nodes/serializers.py @@ -815,6 +815,7 @@ class NodeContributorsCreateSerializer(NodeContributorsSerializer): users = RelationshipField( related_view='users:user-detail', related_view_kwargs={'user_id': ''}, + always_embed=True, required=False ) diff --git a/api_tests/files/views/test_file_detail.py b/api_tests/files/views/test_file_detail.py index 8a478c78c63..893b024341e 100644 --- a/api_tests/files/views/test_file_detail.py +++ b/api_tests/files/views/test_file_detail.py @@ -10,7 +10,7 @@ from api.base.settings.defaults import API_BASE from api_tests import utils as api_utils from framework.auth.core import Auth -from osf.models import NodeLog, Session +from osf.models import NodeLog, Session, QuickFilesNode from osf_tests.factories import ( AuthUserFactory, CommentFactory, @@ -463,6 +463,19 @@ def test_folder_files_relationships_contains_guid_not_id(self, app, user, node): assert node._id in split_href assert node.id not in split_href + def test_embed_user_on_quickfiles_detail(self, app, user): + quickfiles = QuickFilesNode.objects.get(creator=user) + osfstorage = quickfiles.get_addon('osfstorage') + root = osfstorage.get_root() + test_file = root.append_file('speedyfile.txt') + + url = '/{}files/{}/?embed=user'.format(API_BASE, test_file._id) + res = app.get(url, auth=user.auth) + + assert res.json['data'].get('embeds', None) + assert res.json['data']['embeds'].get('user') + assert res.json['data']['embeds']['user']['data']['id'] == user._id + @pytest.mark.django_db class TestFileVersionView: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index cbb392a22bd..8850b90a073 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -105,6 +105,23 @@ services: # volumes: # - emberosf-sync:/code:nocopy +# #################### +# # RabbitMQ SSL +# # Enable this, place the certs in ./ssl, and uncomment the BROKER_USE_SSL dictionary in local.py +# # Uncomment lines under worker in docker-compose +# ##################### +# rabbitmq: +# ports: +# - 5671:5671 +# environment: +# RABBITMQ_SSL_CERTFILE: /etc/ssl/server_certificate.pem +# RABBITMQ_SSL_KEYFILE: /etc/ssl/server_key.pem +# RABBITMQ_SSL_CACERTFILE: /etc/ssl/ca_certificate.pem +# volumes: +# - ./ssl/celery-server.cert.pem:/etc/ssl/server_certificate.pem:ro +# - ./ssl/celery-server.key.pem:/etc/ssl/server_key.pem:ro +# - ./ssl/ca-chain.cert.pem:/etc/ssl/ca_certificate.pem:ro + volumes: osf-sync: external: true diff --git a/docker-compose.yml b/docker-compose.yml index e3f1f7e9936..f0eb7aa250c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ version: '3' volumes: postgres_data_vol: external: false - tokumx_data_vol: + mongo_data_vol: external: false elasticsearch_data_vol: external: false @@ -59,12 +59,12 @@ services: postgres: image: postgres:9.6 - command: |- - /bin/bash -c " - echo \"$$POSTGRES_INITDB\" > /docker-entrypoint-initdb.d/commands.sh && + command: + - /bin/bash + - -c + - echo "$$POSTGRES_INITDB" > /docker-entrypoint-initdb.d/commands.sh && chmod +x /docker-entrypoint-initdb.d/commands.sh && /docker-entrypoint.sh postgres - " ports: - 5432:5432 environment: @@ -76,15 +76,22 @@ services: - "${POSTGRES_DATA_VOL:-postgres_data_vol}:/var/lib/postgresql/data/" stdin_open: true - tokumx: - image: quay.io/centerforopenscience/tokumx:latest - command: mongod --ipv6 + mongo: + image: mongo:3.4 + command: + - /bin/bash + - -c + - openssl req -newkey rsa:2048 -new -x509 -days 9999 -nodes + -subj "/C=US/ST=Virginia/L=Charlottesville/O=COS/OU=Test/CN=192.168.168.167" + -out /etc/ssl/mongodb.crt -keyout /etc/ssl/mongodb.key && + cat /etc/ssl/mongodb.key /etc/ssl/mongodb.crt > /etc/ssl/mongodb.pem && + chown -R mongodb:mongodb /etc/ssl /data/db && + chmod -R 0600 /etc/ssl/* && + gosu mongodb mongod --sslMode requireSSL --sslDisabledProtocols=TLS1_0,TLS1_1 --sslPEMKeyFile /etc/ssl/mongodb.pem ports: - 27017:27017 - environment: - TOKU_HUGE_PAGES_OK: 1 volumes: - - tokumx_data_vol:/data/db + - mongo_data_vol:/data/db stdin_open: true rabbitmq: @@ -113,13 +120,13 @@ services: mfr_requirements: image: quay.io/centerforopenscience/mfr:develop - command: |- - /bin/bash -c " - invoke install --develop && + command: + - /bin/bash + - -c + - invoke install --develop && (python -m compileall /usr/local/lib/python3.5 || true) && rm -Rf /python3.5/* && cp -Rf -p /usr/local/lib/python3.5 / - " restart: 'no' volumes: - mfr_requirements_vol:/python3.5 @@ -144,13 +151,13 @@ services: wb_requirements: image: quay.io/centerforopenscience/waterbutler:develop - command: |- - /bin/bash -c " - invoke install --develop && + command: + - /bin/bash + - -c + - invoke install --develop && (python -m compileall /usr/local/lib/python3.5 || true) && rm -Rf /python3.5/* && cp -Rf -p /usr/local/lib/python3.5 / - " restart: 'no' volumes: - wb_requirements_vol:/python3.5 @@ -230,7 +237,12 @@ services: preprints: image: quay.io/centerforopenscience/ember-preprints:develop-development-local - command: /bin/bash -c "yarn --pure-lockfile --ignore-engines --no-progress --no-emoji && ./node_modules/bower/bin/bower install --allow-root --config.interactive=false && ./node_modules/ember-cli/bin/ember serve --host 0.0.0.0 --port 4200 --live-reload-port 41953" + command: + - /bin/bash + - -c + - yarn --pure-lockfile --ignore-engines --no-progress --no-emoji && + ./node_modules/.bin/bower install --allow-root --config.interactive=false && + ./node_modules/.bin/ember serve --host 0.0.0.0 --port 4200 --live-reload-port 41953 restart: unless-stopped depends_on: - api @@ -253,7 +265,12 @@ services: registries: image: quay.io/centerforopenscience/ember-osf-registries:develop-development-local - command: /bin/bash -c "yarn --pure-lockfile --ignore-engines --no-progress --no-emoji && ./node_modules/bower/bin/bower install --allow-root --config.interactive=false && ./node_modules/ember-cli/bin/ember serve --host 0.0.0.0 --port 4300" + command: + - /bin/bash + - -c + - yarn --pure-lockfile --ignore-engines --no-progress --no-emoji && + ./node_modules/.bin/bower install --allow-root --config.interactive=false && + ./node_modules/.bin/ember serve --host 0.0.0.0 --port 4300 restart: unless-stopped depends_on: - api @@ -273,8 +290,12 @@ services: ############## reviews: - image: quay.io/centerforopenscience/ember-osf-reviews:develop-development-local - command: /bin/bash -c "yarn --pure-lockfile --no-progress --no-emoji && ./node_modules/ember-cli/bin/ember serve --host 0.0.0.0 --port 4400" + image: quay.io/centerforopenscience/osf-reviews:develop + command: + - /bin/bash + - -c + - yarn --pure-lockfile --no-progress --no-emoji && + ./node_modules/.bin/ember serve --host 0.0.0.0 --port 4400 restart: unless-stopped depends_on: - api @@ -297,13 +318,13 @@ services: requirements: image: quay.io/centerforopenscience/osf:develop - command: |- - /bin/bash -c " - invoke requirements --quick && + command: + - /bin/bash + - -c + - invoke requirements --quick && (python -m compileall /usr/local/lib/python2.7 || true) && rm -Rf /python2.7/* && cp -Rf -p /usr/local/lib/python2.7 / - " restart: 'no' environment: DJANGO_SETTINGS_MODULE: api.base.settings @@ -342,7 +363,8 @@ services: ports: - 7007:7007 depends_on: - - tokumx + - mongo + - web env_file: - .docker-compose.sharejs.env volumes: @@ -377,12 +399,16 @@ services: environment: C_FORCE_ROOT: 1 DJANGO_SETTINGS_MODULE: api.base.settings +# BROKER_URL: amqp://guest:guest@192.168.168.167:5671/ env_file: - .docker-compose.env volumes: - osf_requirements_vol:/usr/local/lib/python2.7 - osf_bower_components_vol:/code/website/static/vendor/bower_components - osf_node_modules_vol:/code/node_modules +# - ./ssl/ca-chain.cert.pem:/etc/ssl/certs/ca-chain.cert.pem:ro +# - ./ssl/celery-client.cert.pem:/etc/ssl/certs/worker.pem:ro +# - ./ssl/celery-client.key.pem:/etc/ssl/private/worker.key:ro stdin_open: true admin: @@ -441,6 +467,7 @@ services: DJANGO_SETTINGS_MODULE: api.base.settings env_file: - .docker-compose.env + - .docker-compose.sharejs.env volumes: - osf_requirements_vol:/usr/local/lib/python2.7 - osf_bower_components_vol:/code/website/static/vendor/bower_components diff --git a/framework/celery_tasks/__init__.py b/framework/celery_tasks/__init__.py index e7d1736bfd1..f389437e68a 100644 --- a/framework/celery_tasks/__init__.py +++ b/framework/celery_tasks/__init__.py @@ -17,6 +17,8 @@ client = Client(settings.SENTRY_DSN, release=settings.VERSION, tags={'App': 'celery'}) register_signal(client) +if settings.BROKER_USE_SSL: + app.setup_security() @app.task def error_handler(task_id, task_name): diff --git a/karma.common.conf.js b/karma.common.conf.js index 0d61ded724b..c093c1be1b0 100644 --- a/karma.common.conf.js +++ b/karma.common.conf.js @@ -32,6 +32,9 @@ var webpackTestConfig = { // Assume test files are ES6 {test: /\.test\.js$/, loader: 'babel-loader'}, ]) + }, + node: { + fs: 'empty' } }; diff --git a/package.json b/package.json index b312ee9eb1f..d680cb737f0 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "lodash.includes": "^4.3.0", "lodash.set": "^4.3.2", "markdown-it": "~4.1.0", + "markdown-it-imsize": "^2.0.1", "markdown-it-ins-del": "^0.1.1", "markdown-it-sanitizer": "~0.3.0", "@centerforopenscience/markdown-it-toc": "~1.1.1", diff --git a/webpack.common.config.js b/webpack.common.config.js index 780f63603e0..0460b0a83d4 100644 --- a/webpack.common.config.js +++ b/webpack.common.config.js @@ -222,5 +222,8 @@ module.exports = { //Dirty hack because mime-type's json file is "special" {test: /db.json/, loader: 'json-loader'} ] + }, + node: { + fs: 'empty' } }; diff --git a/website/settings/defaults.py b/website/settings/defaults.py index eb69cb0a70e..4e02f8b28e4 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -414,6 +414,7 @@ def parent_dir(path): RABBITMQ_VHOST = os.environ.get('RABBITMQ_VHOST', '/') BROKER_URL = os.environ.get('BROKER_URL', 'amqp://{}:{}@{}:{}/{}'.format(RABBITMQ_USERNAME, RABBITMQ_PASSWORD, RABBITMQ_HOST, RABBITMQ_PORT, RABBITMQ_VHOST)) +BROKER_USE_SSL = False # Default RabbitMQ backend CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', BROKER_URL) diff --git a/website/settings/local-dist.py b/website/settings/local-dist.py index 505dd6747db..516a4dae075 100644 --- a/website/settings/local-dist.py +++ b/website/settings/local-dist.py @@ -85,6 +85,16 @@ ## Default RabbitMQ broker BROKER_URL = 'amqp://' +# Celery with SSL +# import ssl +# +# BROKER_USE_SSL = { +# 'keyfile': '/etc/ssl/private/worker.key', +# 'certfile': '/etc/ssl/certs/worker.pem', +# 'ca_certs': '/etc/ssl/certs/ca-chain.cert.pem', +# 'cert_reqs': ssl.CERT_REQUIRED, +# } + # Default RabbitMQ backend CELERY_RESULT_BACKEND = 'amqp://' diff --git a/website/static/js/markdown.js b/website/static/js/markdown.js index 957a756a640..5d2229debc0 100644 --- a/website/static/js/markdown.js +++ b/website/static/js/markdown.js @@ -33,32 +33,39 @@ var oldMarkdownList = function(md) { // Full markdown renderer for views / wiki pages / pauses between typing var markdown = new MarkdownIt('commonmark', { - highlight: highlighter + highlight: highlighter, + linkify: true }) .use(require('markdown-it-video')) .use(require('@centerforopenscience/markdown-it-toc')) .use(require('markdown-it-sanitizer')) + .use(require('markdown-it-imsize')) .use(insDel) .enable('table') + .enable('linkify') .use(bootstrapTable) .disable('strikethrough'); // Fast markdown renderer for active editing to prevent slow loading/rendering tasks -var markdownQuick = new MarkdownIt(('commonmark'), { }) +var markdownQuick = new MarkdownIt('commonmark', { linkify: true }) .use(require('markdown-it-sanitizer')) + .use(require('markdown-it-imsize')) .disable('link') .disable('image') .use(insDel) .enable('table') + .enable('linkify') .use(bootstrapTable) .disable('strikethrough'); // Markdown renderer for older wikis rendered before switch date -var markdownOld = new MarkdownIt(('commonmark'), { }) +var markdownOld = new MarkdownIt('commonmark', { linkify: true}) .use(require('markdown-it-sanitizer')) + .use(require('markdown-it-imsize')) .use(insDel) .enable('table') + .enable('linkify') .use(bootstrapTable) .use(oldMarkdownList) .disable('strikethrough'); diff --git a/website/static/js/nodesDeleteTreebeard.js b/website/static/js/nodesDeleteTreebeard.js index 5a4bf74d7e5..46f0b429aba 100644 --- a/website/static/js/nodesDeleteTreebeard.js +++ b/website/static/js/nodesDeleteTreebeard.js @@ -64,7 +64,7 @@ function NodesDeleteTreebeard(divID, data, nodesState, nodesOriginal) { } ]; }, - onload : function () { + ondataload : function () { var tb = this; expandOnLoad.call(tb); }, diff --git a/yarn.lock b/yarn.lock index 1719658d5e2..d1a09be58ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2427,6 +2427,10 @@ lru-cache@~1.0.2: version "1.0.6" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-1.0.6.tgz#aa50f97047422ac72543bda177a9c9d018d98452" +markdown-it-imsize@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/markdown-it-imsize/-/markdown-it-imsize-2.0.1.tgz#cca0427905d05338a247cb9ca9d968c5cddd5170" + markdown-it-ins-del@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/markdown-it-ins-del/-/markdown-it-ins-del-0.1.1.tgz#8b421d8e788ee84a1d572eb3fcc0028ce431c9ea"