Use Elasticsearch 6.x

psapoznik · Oct 30, 2018 · 44aba0d · 44aba0d
1 parent baeefb8
commit 44aba0d
Show file tree

Hide file tree

Showing 15 changed files with 1,450 additions and 34 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -11,8 +11,9 @@ RUN \
 # Install Python dependencies
 RUN pip install pipenv
 ADD Pipfile /app/
+ADD Pipfile.lock /app/
 WORKDIR /app
-RUN pipenv install --system --deploy --dev --skip-lock
+RUN pipenv install --system --deploy --dev
 
 ### Build static assets
 FROM node:10 as build-nodejs

diff --git a/Pipfile b/Pipfile
@@ -35,13 +35,13 @@ django-templated-email = ">=2.2.0"
 django-versatileimagefield = ">=1.10"
 django-webpack-loader = ">=0.3.0"
 django-celery-results = "*"
-django-elasticsearch-dsl = "*"
+django-elasticsearch-dsl = ">=0.5.0.<0.6.0"
 django-recaptcha = "*"
 django-silk = "==2.0.0"
 django-debug-toolbar = "*"
 django-debug-toolbar-request-history = "*"
-elasticsearch = ">=5.4,<6.0.0"
-elasticsearch-dsl = ">=5.0.0,<6.0.0"
+elasticsearch = ">=6.0.0,<6.1.0"
+elasticsearch-dsl = ">=6.0.0,<6.1.0"
 faker = ">=0.7.7"
 freezegun = ">=0.3.9"
 google-measurement-protocol = ">=1.0.0"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -59,7 +59,7 @@ services:
       - redis
 
   search:
-    image: elasticsearch:5.4.3
+    image: elasticsearch:6.4.2
     restart: unless-stopped
     networks:
       - saleor-backend-tier
@@ -70,7 +70,7 @@ services:
     # See https://github.com/docker/compose/issues/4513 if updating to version '3'
     mem_limit: 512m
     environment:
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.disable.jmx=true"
 
 volumes:
   saleor-db:

diff --git a/requirements.txt b/requirements.txt
@@ -51,8 +51,8 @@ django-versatileimagefield==1.10
 django-webpack-loader==0.6.0
 django==2.1.2
 docutils==0.14
-elasticsearch-dsl==5.4.0
-elasticsearch==5.5.3
+elasticsearch-dsl==6.0.1
+elasticsearch==6.0.0
 faker==0.9.1
 freezegun==0.3.10
 google-i18n-address==2.3.2

diff --git a/saleor/search/backends/test_elasticsearch_storefront.py b/saleor/search/backends/test_elasticsearch_storefront.py
@@ -3,7 +3,7 @@
 PHRASE = 'How fortunate man with none'
 FIELDS = ['title', 'name', 'description']
 INDEX = 'storefront'
-DOC_TYPE = 'product_document'
+DOC_TYPE = 'doc'
 QUERY = {
     '_source': False,
     'query': {

diff --git a/tests/cassettes/test_dashboard_search_orders_by_user.yaml b/tests/cassettes/test_dashboard_search_orders_by_user.yaml
@@ -6,7 +6,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
     body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -22,9 +22,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/orders/order_document/_search
+    uri: http://search:9200/orders/doc/_search
   response:
-    body: {string: '{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":2,"max_score":2.1282318,"hits":[{"_index":"orders","_type":"order_document","_id":"19","_score":2.1282318,"_source":{}},{"_index":"orders","_type":"order_document","_id":"18","_score":2.1282318,"_source":{}}]}}'}
+    body: {string: '{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":2,"max_score":2.1282318,"hits":[{"_index":"orders","_type":"doc","_id":"19","_score":2.1282318,"_source":{}},{"_index":"orders","_type":"doc","_id":"18","_score":2.1282318,"_source":{}}]}}'}
     headers:
       Warning: ['299 Elasticsearch-5.4.3-eed30a8 "Deprecated field [exclude] used,
           expected [excludes] instead" "Thu, 16 Nov 2017 09:44:26 GMT"']
@@ -39,9 +39,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/users/user_document/_search
+    uri: http://search:9200/users/doc/_search
   response:
-    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":5.115776,"hits":[{"_index":"users","_type":"user_document","_id":"17","_score":5.115776,"_source":{}}]}}'}
+    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":5.115776,"hits":[{"_index":"users","_type":"doc","_id":"17","_score":5.115776,"_source":{}}]}}'}
     headers:
       Warning: ['299 Elasticsearch-5.4.3-eed30a8 "Deprecated field [exclude] used,
           expected [excludes] instead" "Thu, 16 Nov 2017 09:44:26 GMT"']

diff --git a/tests/cassettes/test_dashboard_search_user_by_email.yaml b/tests/cassettes/test_dashboard_search_user_by_email.yaml
@@ -6,7 +6,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
     body: {string: '{"took":0,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -22,7 +22,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/orders/order_document/_search
+    uri: http://search:9200/orders/doc/_search
   response:
     body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -39,9 +39,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/users/user_document/_search
+    uri: http://search:9200/users/doc/_search
   response:
-    body: {string: '{"took":3,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":5.115776,"hits":[{"_index":"users","_type":"user_document","_id":"9","_score":5.115776,"_source":{}}]}}'}
+    body: {string: '{"took":3,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":5.115776,"hits":[{"_index":"users","_type":"doc","_id":"9","_score":5.115776,"_source":{}}]}}'}
     headers:
       Warning: ['299 Elasticsearch-5.4.3-eed30a8 "Deprecated field [exclude] used,
           expected [excludes] instead" "Thu, 16 Nov 2017 09:44:26 GMT"']

diff --git a/tests/cassettes/test_dashboard_search_user_name.yaml b/tests/cassettes/test_dashboard_search_user_name.yaml
@@ -6,7 +6,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/orders/order_document/_search
+    uri: http://search:9200/orders/doc/_search
   response:
     body: {string: '{"took":0,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -22,7 +22,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
     body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -39,9 +39,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/users/user_document/_search
+    uri: http://search:9200/users/doc/_search
   response:
-    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":2.6855774,"hits":[{"_index":"users","_type":"user_document","_id":"6","_score":2.6855774,"_source":{}}]}}'}
+    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":2.6855774,"hits":[{"_index":"users","_type":"doc","_id":"6","_score":2.6855774,"_source":{}}]}}'}
     headers:
       Warning: ['299 Elasticsearch-5.4.3-eed30a8 "Deprecated field [exclude] used,
           expected [excludes] instead" "Thu, 16 Nov 2017 09:37:53 GMT"']

diff --git a/tests/cassettes/test_dashboard_search_with_empty_results.yaml b/tests/cassettes/test_dashboard_search_with_empty_results.yaml
@@ -6,7 +6,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
     body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -22,7 +22,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/orders/order_document/_search
+    uri: http://search:9200/orders/doc/_search
   response:
     body: {string: '{"took":0,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -39,7 +39,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/users/user_document/_search
+    uri: http://search:9200/users/doc/_search
   response:
     body: {string: '{"took":3,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:

diff --git a/tests/cassettes/test_dashboard_search_with_product_result.yaml b/tests/cassettes/test_dashboard_search_with_product_result.yaml
@@ -6,9 +6,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
-    body: {string: '{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":3,"max_score":2.9128819,"hits":[{"_index":"storefront","_type":"product_document","_id":"23","_score":2.9128819,"_source":{}},{"_index":"storefront","_type":"product_document","_id":"58","_score":2.9128819,"_source":{}},{"_index":"storefront","_type":"product_document","_id":"56","_score":2.9128819,"_source":{}}]}}'}
+    body: {string: '{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":3,"max_score":2.9128819,"hits":[{"_index":"storefront","_type":"doc","_id":"23","_score":2.9128819,"_source":{}},{"_index":"storefront","_type":"doc","_id":"58","_score":2.9128819,"_source":{}},{"_index":"storefront","_type":"doc","_id":"56","_score":2.9128819,"_source":{}}]}}'}
     headers:
       Warning: ['299 Elasticsearch-5.4.3-eed30a8 "Deprecated field [exclude] used,
           expected [excludes] instead" "Thu, 16 Nov 2017 09:44:25 GMT"']
@@ -22,7 +22,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/orders/order_document/_search
+    uri: http://search:9200/orders/doc/_search
   response:
     body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:
@@ -39,7 +39,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/users/user_document/_search
+    uri: http://search:9200/users/doc/_search
   response:
     body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:

diff --git a/tests/cassettes/test_new_search_doesnt_show_unpublished.yaml b/tests/cassettes/test_new_search_doesnt_show_unpublished.yaml
@@ -7,9 +7,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
-    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":2,"max_score":3.223138,"hits":[{"_index":"storefront","_type":"product_document","_id":"15","_score":3.223138},{"_index":"storefront","_type":"product_document","_id":"56","_score":3.223138}]}}'}
+    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":2,"max_score":3.223138,"hits":[{"_index":"storefront","_type":"doc","_id":"15","_score":3.223138},{"_index":"storefront","_type":"doc","_id":"56","_score":3.223138}]}}'}
     headers:
       content-length: ['285']
       content-type: [application/json; charset=UTF-8]

diff --git a/tests/cassettes/test_new_search_with_empty_results.yaml b/tests/cassettes/test_new_search_with_empty_results.yaml
@@ -7,7 +7,7 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
     body: {string: '{"took":0,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'}
     headers:

diff --git a/tests/cassettes/test_new_search_with_result.yaml b/tests/cassettes/test_new_search_with_result.yaml
@@ -7,9 +7,9 @@ interactions:
       connection: [keep-alive]
       content-type: [application/json]
     method: GET
-    uri: http://search:9200/storefront/product_document/_search
+    uri: http://search:9200/storefront/doc/_search
   response:
-    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":2,"max_score":3.223138,"hits":[{"_index":"storefront","_type":"product_document","_id":"15","_score":3.223138},{"_index":"storefront","_type":"product_document","_id":"56","_score":3.223138}]}}'}
+    body: {string: '{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":2,"max_score":3.223138,"hits":[{"_index":"storefront","_type":"doc","_id":"15","_score":3.223138},{"_index":"storefront","_type":"doc","_id":"56","_score":3.223138}]}}'}
     headers:
       content-length: ['285']
       content-type: [application/json; charset=UTF-8]

diff --git a/tests/settings.py b/tests/settings.py
@@ -7,6 +7,10 @@
 
 LANGUAGE_CODE = 'en'
 
+ES_URL = None
+SEARCH_BACKEND = 'saleor.search.backends.postgresql'
+INSTALLED_APPS = [a for a in INSTALLED_APPS if a != 'django_elasticsearch_dsl']
+
 RECAPTCHA_PUBLIC_KEY = None
 RECAPTCHA_PRIVATE_KEY = None