Merge pull request #1090 from macarthur-lab/dev

Dev

deploy/MIGRATE.md

@@ -1,8 +1,4 @@
-This README describes steps for migrating an older xBrowse instance. 
-
-seqr is still not ready for an official major release, but the database schema and loading pipelines have 
-stabilized so anyone that wants migrate an older xBrowse instance to the current seqr instance can do so by 
-following these steps:
+This README describes steps for migrating an older xBrowse or seqr instance.
 
 1. Backup your current SQL database:
 
@@ -13,21 +9,34 @@ following these steps:
 2. Download or clone the lastest seqr code from [https://github.com/macarthur-lab/seqr](https://github.com/macarthur-lab/seqr)
 
 3. Run migrations:
-
    ```
    python2.7 -m manage makemigrations 
    python2.7 -m manage migrate 
+   python2.7 -m manage loaddata variant_tag_types // This will fail if it has been run before, and that is okay
+   python2.7 -m manage loaddata variant_searches // This will fail if it has been run before, and that is okay
    ```
-
-4. Copy existing data to the new database tables:
+
+4. If you were previously on a version of seqr that used mongo instead of elasticsearch to store variant data, 
+load new datasets into elasticsearch using the hail-based pipelines described in the main README. 
+If you were previously using elasticsearch, run:
     ```
-    python2.7 -m manage transfer_gene_lists
-    python2.7 -m manage update_projects_in_new_schema
     python2.7 -m manage reload_saved_variant_json
     ```
 
 5. Update gene-level reference datasets:
     ```
-    python2.7 -m manage update_all_reference_data --omim-key $(cat deploy/secrets/minikube/seqr/omim_key)
+    psql -U postgres postgres -c "drop database reference_data_db"
+    psql -U postgres postgres -c "create database reference_data_db"
+    REFERENCE_DATA_BACKUP_FILE=gene_reference_data_backup.gz
+    wget -N https://storage.googleapis.com/seqr-reference-data/gene_reference_data_backup.gz -O ${REFERENCE_DATA_BACKUP_FILE}
+    psql -U postgres reference_data_db <  <(gunzip -c ${REFERENCE_DATA_BACKUP_FILE})
+    rm ${REFERENCE_DATA_BACKUP_FILE}
     ```
-6. Load new datasets into elasticsearch using the hail-based pipelines described in the main README.
+    
+6. If after relaunching seqr you do not see any projects on the main page, check if your data is available on the 
+deprecated pages (http://localhost:8000/projects). If so run the following to transfer to the new pages:
+    ```
+    python2.7 -m manage update_projects_in_new_schema
+    python2.7 -m manage transfer_gene_lists
+    python2.7 -m manage transfer_gene_notes
+    ```

seqr/views/react_app.py

@@ -1,12 +1,12 @@
 import json
 import re
-import urllib
 from django.contrib.auth.models import User
 from django.contrib.auth.decorators import login_required
 from django.core.serializers.json import DjangoJSONEncoder
 from django.template import loader
 from django.http import HttpResponse
 
+from settings import SEQR_VERSION
 from seqr.views.utils.orm_to_json_utils import _get_json_for_user
 
 
@@ -28,6 +28,7 @@ def no_login_main_app(request, *args, **kwargs):
 
 
 def _render_app_html(request, initial_json):
+    initial_json['meta'] = {'version': SEQR_VERSION}
     html = loader.render_to_string('app.html')
 
     html = html.replace(

seqr/views/utils/pedigree_info_utils.py

@@ -429,6 +429,9 @@ def _get_datstat_family_notes(row):
     def _get_column_val(column):
         return DC.VALUE_MAP[column][row[column]]
 
+    def _get_list_column_val(column):
+        return ', '.join([DC.VALUE_MAP[column][raw_val] for raw_val in row[column].split(',')])
+
     def _has_test(test):
         return row['TESTS.{}'.format(test)] == DC.YES
 
@@ -499,7 +502,6 @@ def _bool_condition_val(val, display, unknown_display):
     clinical_diagnoses = _get_column_val(DC.CLINICAL_DIAGNOSES_COLUMN)
     genetic_diagnoses = _get_column_val(DC.GENETIC_DIAGNOSES_COLUMN)
     doctors_list = json.loads(row[DC.DOCTOR_TYPES_COLUMN])
-    biopsy_split = row[DC.BIOPSY_COLUMN].split(': ')
 
     if _has_test(DC.NONE_TEST):
         testing = 'None'
@@ -572,8 +574,8 @@ def _bool_condition_val(val, display, unknown_display):
         doctors=', '.join(doctors_list).replace('ClinGen', 'Clinical geneticist'),
         other_doctors=u': {}'.format(row[DC.DOCTOR_TYPES_SPECIFY_COLUMN] or 'Unspecified') if 'Other' in doctors_list else '',
         testing=testing,
-        biopses='None' if 'NONE' in biopsy_split[0] or not biopsy_split[0] else biopsy_split[1],
-        other_biopses=u': {}'.format(row[DC.OTHER_BIOPSY_COLUMN] or 'Unspecified') if 'OTHER' in biopsy_split[0] else '',
+        biopses='None' if (row[DC.NO_BIOPSY_COLUMN] == DC.YES or not row[DC.BIOPSY_COLUMN]) else _get_list_column_val(DC.BIOPSY_COLUMN),
+        other_biopses=u': {}'.format(row[DC.OTHER_BIOPSY_COLUMN] or 'Unspecified') if 'OTHER' in row[DC.BIOPSY_COLUMN] else '',
         studies=u'Yes, Name of studies: {study_names}, Expecting results: {expecting_results}'.format(
             study_names=row[DC.OTHER_STUDIES_COLUMN] or 'Unspecified',
             expecting_results=_get_column_val(DC.EXPECTING_RESULTS_COLUMN) if row[DC.EXPECTING_RESULTS_COLUMN] else 'Unspecified',
@@ -717,6 +719,7 @@ class DatstatConstants:
     MICROARRAY_RELATIVE_SPEC_COLUMN = 'TESTS_MICROARRAY_RELATIVE_SPEC'
     OTHER_TEST_COLUMN = 'TEST_OTHER_SPECIFY'
     BIOPSY_COLUMN = 'BIOPSY'
+    NO_BIOPSY_COLUMN = 'BIOPSY.NONE'
     OTHER_BIOPSY_COLUMN = 'BIOPSY_OTHER_SPECIFY'
     HAS_OTHER_STUDIES_COLUMN = 'OTHER_GENETIC_STUDIES'
     OTHER_STUDIES_COLUMN = 'OTHER_GENETIC_STUDIES_SPECIFY'
@@ -725,13 +728,19 @@ class DatstatConstants:
     SEX_OPTION_MAP = {'1': 'MALE', '2': 'FEMALE', '3': 'UNKNOWN'}
     ETHNICITY_COLUMN_MAP = {'1': 'Hispanic', '2': 'Not Hispanic', '3': 'Unknown', '4': 'I prefer not to answer'}
     SAMPLE_AVAILABILITY_MAP = {'1': 'available', '2': 'not available', '3': 'availability unknown'}
+    BIOPSY_MAP = {
+        biopsy_type: '{} Biopsy'.format(biopsy_type.replace('_', ' ').title())
+        for biopsy_type in ['MUSCLE', 'BONE_MARROW', 'LIVER', 'HEART', 'SKIN', 'CRANIOFACIAL']
+    }
+    BIOPSY_MAP['OTHER'] = 'Other Tissue Biopsy'
 
     VALUE_MAP = {
         CLINICAL_DIAGNOSES_COLUMN: YES_NO_UNSURE_MAP,
         GENETIC_DIAGNOSES_COLUMN: YES_NO_UNSURE_MAP,
         ETHNICITY_COLUMN: ETHNICITY_COLUMN_MAP,
         EXPECTING_RESULTS_COLUMN: YES_NO_UNSURE_MAP,
         SAMPLE_AVAILABILITY_COLUMN: SAMPLE_AVAILABILITY_MAP,
+        BIOPSY_COLUMN: BIOPSY_MAP
     }
 
     OTHER_RELATIONSHIP_CODE = '6'

seqr/views/utils/pedigree_info_utils_tests.py

@@ -76,8 +76,8 @@ def test_parse_pedigree_table(self):
     def test_parse_datstat_pedigree_table(self):
         records, errors, warnings = parse_pedigree_table(
         [['DATSTAT_ALTPID', 'FAMILY_ID', 'DDP_CREATED', 'DDP_LASTUPDATED', 'RELATIONSHIP', 'RELATIONSHIP_SPECIFY', 'PATIENT_WEBSITE', 'DESCRIPTION', 'CLINICAL_DIAGNOSES', 'CLINICAL_DIAGNOSES_SPECIFY', 'GENETIC_DIAGNOSES', 'GENETIC_DIAGNOSES_SPECIFY', 'FIND_OUT.DOCTOR', 'FIND_OUT_DOCTOR_DETAILS', 'PATIENT_AGE', 'CONDITION_AGE', 'PATIENT_DECEASED', 'DECEASED_AGE', 'DECEASED_CAUSE', 'DECEASED_STORED_SAMPLE', 'PATIENT_SEX', 'RACE_LIST', 'PTETHNICITY', 'DOCTOR_TYPES_LIST', 'DOCTOR_TYPES_SPECIFY', 'TESTS.NONE', 'TESTS.NOT_SURE', 'TESTS.KARYOTYPE', 'TESTS.SINGLE_GENE_TESTING', 'TESTS.GENE_PANEL_TESTING', 'TESTS.MITOCHON_GENOME_SEQUENCING', 'TESTS.MICROARRAY', 'TESTS_MICROARRAY_YEAR', 'TESTS_MICROARRAY_LAB', 'TESTS_MICROARRAY_RELATIVE_LIST', 'TESTS_MICROARRAY_RELATIVE_SPEC', 'TESTS.WEXOME_SEQUENCING', 'TESTS_WEXOME_SEQUENCING_YEAR', 'TESTS_WEXOME_SEQUENCING_LAB', 'TESTS_WEXOME_SEQUENCING_REL_LI', 'TESTS_WEXOME_SEQUENCING_REL_SP', 'TESTS.WGENOME_SEQUENCING', 'TESTS_WGENOME_SEQUENCING_YEAR', 'TESTS_WGENOME_SEQUENCING_LAB', 'TESTS_WGENOME_SEQUENCING_REL_L', 'TESTS_WGENOME_SEQUENCING_REL_S', 'TESTS.OTHER', 'TEST_OTHER_SPECIFY', 'BIOPSY.NONE', 'BIOPSY', 'BIOPSY.OTHER', 'BIOPSY_OTHER_SPECIFY', 'OTHER_GENETIC_STUDIES', 'OTHER_GENETIC_STUDIES_SPECIFY', 'EXPECTING_GENETIC_RESULTS', 'SAME_CONDITION_MOM', 'CONDITION_AGE_MOM', 'ABLE_TO_PARTICIPATE_MOM', 'DECEASED_MOM', 'STORED_DNA_MOM', 'SAME_CONDITION_DAD', 'CONDITION_AGE_DAD', 'ABLE_TO_PARTICIPATE_DAD', 'DECEASED_DAD', 'STORED_DNA_DAD', 'NO_SIBLINGS', 'SIBLING_LIST', 'NO_CHILDREN', 'CHILD_LIST', 'NO_RELATIVE_AFFECTED', 'RELATIVE_LIST', 'FAMILY_INFO'],
-        ['1518231365', '123', '2019-07-31T03:54:21UTC', '2019-08-01T14:12:40UTC', '6', 'Grandchild', 'wwww.myblog.com', u'I have a really debilitating probably genetic condition. I\xe2ve seen many specialists.', '1', u'SMA\xe2s', '1', u'Dwarfism\xe2', '1', 'Dr John Smith', '34', '21', '1', '33', 'heart attack', '2', '1', '["White","Asian","Pacific"]', '2', '["ClinGen","Neurologist","Cardiologist","Other"]', 'Pediatrician', '0', '0', '0', '1', '1', '0', '0', '', '', '', '', '1', '2018', u'UDN\xe2s lab', '["Parent","AuntUncle","NieceNephew","Other"]', 'Grandmother', '1', '', '', '', 'Grandmother', '1', 'Blood work', '0', 'MUSCLE,SKIN,OTHER: Muscle Biopsy, Skin Biopsy, Other Tissue Biopsy', '1', u'Bone\xe2s', '1', 'Undiagnosed Diseases Network', '2', '1', '19', '1', '', '', '2', '', '', '1', '2', '0', '[{"sex":"Female","age":"21","races":["White"],"ethnicity":"NonHispanic","sameCondition":"Yes","ageOnsetCondition":null,"ableToParticipate":"No","siblingId":"d18b9f4b-0995-45e9-9b00-e710d0004a3f"},{"sex":"","age":"17","races":["White"],"ethnicity":"NonHispanic","sameCondition":"","ageOnsetCondition":null,"ableToParticipate":"Yes","siblingId":"3ddc9015-3c2c-484c-b1de-502ba9ffc1e4"}]', '1', '', '0', '[{"sex":"Male","age":"44","races":["White"],"ethnicity":"NonHispanic","sameCondition":"No","ageOnsetCondition":null,"ableToParticipate":"","siblingId":"bb87c69f-6c52-48b4-8854-e639d998abe7"}]', u'patient\xe2s uncle (dads brother) died from Fahrs disease at 70'],
-        ['b392fd78b440', '987', '2019-08-06T14:30:44UTC', '2019-08-06T15:18:48UTC', '2', 'Grandchild', '', '', '3', 'SMA', '2', 'Dwarfism', '0', 'Dr John Smith', '47', '2', '0', '33', 'heart attack', '2', '3', '["White"]', '3', '[]', 'Pediatrician', '0', '1', '0', '1', '1', '0', '0', '', '', '', '', '1', '2018', 'UDN', '["Parent","AuntUncle","NieceNephew","Other"]', 'Grandmother', '1', '', '', '', 'Grandmother', '1', 'Blood work', '0', 'NONE: This individual hasn\'t had a biopsy', '1', 'Bone', '0', 'Undiagnosed Diseases Network', '2', '3', '19', '2', '3', '', '', '', '', '', '1', '1', '[{"sex":"Female","age":"21","races":["White"],"ethnicity":"NonHispanic","sameCondition":"Yes","ageOnsetCondition":null,"ableToParticipate":"No","siblingId":"d18b9f4b-0995-45e9-9b00-e710d0004a3f"},{"sex":"","age":"17","races":["White"],"ethnicity":"NonHispanic","sameCondition":"","ageOnsetCondition":null,"ableToParticipate":"Yes","siblingId":"3ddc9015-3c2c-484c-b1de-502ba9ffc1e4"}]', '0: No', '[{"sex":"Male","age":"12","races":["White"],"ethnicity":"NonHispanic","sameCondition":"No","ageOnsetCondition":null,"ableToParticipate":"Unsure","siblingId":"bb87c69f-6c52-48b4-8854-e639d998abe7"}]', '1', '', '']], FILENAME)
+        ['1518231365', '123', '2019-07-31T03:54:21UTC', '2019-08-01T14:12:40UTC', '6', 'Grandchild', 'wwww.myblog.com', u'I have a really debilitating probably genetic condition. I\xe2ve seen many specialists.', '1', u'SMA\xe2s', '1', u'Dwarfism\xe2', '1', 'Dr John Smith', '34', '21', '1', '33', 'heart attack', '2', '1', '["White","Asian","Pacific"]', '2', '["ClinGen","Neurologist","Cardiologist","Other"]', 'Pediatrician', '0', '0', '0', '1', '1', '0', '0', '', '', '', '', '1', '2018', u'UDN\xe2s lab', '["Parent","AuntUncle","NieceNephew","Other"]', 'Grandmother', '1', '', '', '', 'Grandmother', '1', 'Blood work', '0', 'MUSCLE,SKIN,OTHER', '1', u'Bone\xe2s', '1', 'Undiagnosed Diseases Network', '2', '1', '19', '1', '', '', '2', '', '', '1', '2', '0', '[{"sex":"Female","age":"21","races":["White"],"ethnicity":"NonHispanic","sameCondition":"Yes","ageOnsetCondition":null,"ableToParticipate":"No","siblingId":"d18b9f4b-0995-45e9-9b00-e710d0004a3f"},{"sex":"","age":"17","races":["White"],"ethnicity":"NonHispanic","sameCondition":"","ageOnsetCondition":null,"ableToParticipate":"Yes","siblingId":"3ddc9015-3c2c-484c-b1de-502ba9ffc1e4"}]', '1', '', '0', '[{"sex":"Male","age":"44","races":["White"],"ethnicity":"NonHispanic","sameCondition":"No","ageOnsetCondition":null,"ableToParticipate":"","siblingId":"bb87c69f-6c52-48b4-8854-e639d998abe7"}]', u'patient\xe2s uncle (dads brother) died from Fahrs disease at 70'],
+        ['b392fd78b440', '987', '2019-08-06T14:30:44UTC', '2019-08-06T15:18:48UTC', '2', 'Grandchild', '', '', '3', 'SMA', '2', 'Dwarfism', '0', 'Dr John Smith', '47', '2', '0', '33', 'heart attack', '2', '3', '["White"]', '3', '[]', 'Pediatrician', '0', '1', '0', '1', '1', '0', '0', '', '', '', '', '1', '2018', 'UDN', '["Parent","AuntUncle","NieceNephew","Other"]', 'Grandmother', '1', '', '', '', 'Grandmother', '1', 'Blood work', '1', 'NONE: This individual hasn\'t had a biopsy', '1', 'Bone', '0', 'Undiagnosed Diseases Network', '2', '3', '19', '2', '3', '', '', '', '', '', '1', '1', '[{"sex":"Female","age":"21","races":["White"],"ethnicity":"NonHispanic","sameCondition":"Yes","ageOnsetCondition":null,"ableToParticipate":"No","siblingId":"d18b9f4b-0995-45e9-9b00-e710d0004a3f"},{"sex":"","age":"17","races":["White"],"ethnicity":"NonHispanic","sameCondition":"","ageOnsetCondition":null,"ableToParticipate":"Yes","siblingId":"3ddc9015-3c2c-484c-b1de-502ba9ffc1e4"}]', '0: No', '[{"sex":"Male","age":"12","races":["White"],"ethnicity":"NonHispanic","sameCondition":"No","ageOnsetCondition":null,"ableToParticipate":"Unsure","siblingId":"bb87c69f-6c52-48b4-8854-e639d998abe7"}]', '1', '', '']], FILENAME)
 
         note_1 = u"""#### Clinical Information
       __Patient is my:__ Grandchild (male)

settings.py

@@ -5,6 +5,8 @@
 
 logger = logging.getLogger(__name__)
 
+SEQR_VERSION = 'v0.1'
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))