for local test

cirDraw/cirDraw/settings.py

@@ -84,11 +84,13 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',
+        #'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'dj_db_conn_pool.backends.mysql',
         'NAME': 'circDraw',
         'USER': 'root',
         'PASSWORD': 'mypassword',
-        'HOST': 'localhost',
+        #'HOST': '167.179.90.87',
+        'HOST': '127.0.0.1',
         'PORT': 6603,
         'OPTIONS': {'local_infile': 1}
     }

cirDraw/docker/nginx/Dockerfile

@@ -0,0 +1,8 @@
+FROM nginx:latest
+
+COPY cirDraw_nginx.conf /etc/nginx/sites-available/
+
+RUN mkdir -p /etc/nginx/sites-enabled/\
+    && ln -s /etc/nginx/sites-available/cirDraw_nginx.conf /etc/nginx/sites-enabled/
+
+CMD ["nginx", "-g", "daemon off;"]

cirDraw/docker/nginx/cirDraw_nginx.conf

@@ -0,0 +1,43 @@
+# mysite_nginx.conf
+
+# the upstream component nginx needs to connect to
+upstream django {
+    # server unix:///docker_api/circDraw.sock
+    server web:8888; # for a web port socket (we'll use this first)
+}
+
+
+
+
+# default server
+# server {
+#    return 404;
+#}
+
+# configuration of the server
+server {
+    # the port your site will be served on
+    listen      80 default_server;
+    # the domain name it will serve for
+    server_name www.circdraw.com; # substitute your machine's IP address or FQDN
+    charset     utf-8;
+
+    # max upload size
+    client_max_body_size 200M;   # adjust to taste
+
+    # Django media
+    location /media  {
+        alias /docker_api/media;  # your Django project's media files - amend as required
+    }
+
+    location /static {
+        alias  /docker_api/static; # your Django project's static files - amend as required
+    }
+
+    # Finally, send all non-media requests to the Django server.
+    location / {
+        uwsgi_pass  django;
+        include     /docker_api/uwsgi_conf/uwsgi_params; # the uwsgi_params file you installed
+    }
+
+}

cirDraw/docker/uwsgi_conf/uwsgi.ini

@@ -0,0 +1,28 @@
+# mysite_uwsgi.ini file
+[uwsgi]
+
+# Django-related settings
+# the base directory (full path)
+
+http-socket = :8000
+
+chdir           = /docker_api/cirDraw/
+# Django's wsgi file
+module          = cirDraw.wsgi
+# the virtualenv (full path)
+#home            = /root/venv-circDraw/
+
+# process-related settings
+# master
+master          = true
+# maximum number of worker processes
+processes       = 10
+# the socket (use the full path to be safe
+socket          = /docker_api/circDraw.sock
+# ... with appropriate permissions - may be needed
+chmod-socket    = 666
+# clear environment on exit
+vacuum          = true
+
+# log 
+logger = file:/tmp/errlog_uwsgi

cirDraw/docker/uwsgi_conf/uwsgi_params

@@ -0,0 +1,17 @@
+
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;

cirDraw/tools/handle_file.py

@@ -1,19 +1,22 @@
-import pymysql
 import os
 import re
 import ujson
 from intervaltree import IntervalTree
 import multiprocessing as mp
-from time import sleep
+import time 
 from .models import *
-from django.db import connection
+#from django.db import connection
 from collections import Counter
+import pymysql
+import sqlalchemy
+from sqlalchemy.pool import QueuePool
 
+#engine = sqlalchemy.create_engine('mysql+pymysql://root:[email protected]:6603/circDraw', poolclass=QueuePool)
+engine = sqlalchemy.create_engine('mysql+pymysql://root:[email protected]:6603/circDraw', poolclass=QueuePool)
+print('Running handle')
 
 def line_counter(file):
     line_count = 0
-
-
     with open(file,'r') as f:
         for line in f:
             line_count += 1
@@ -152,6 +155,8 @@ def find_exon_combo(chr_num: str, circStart: int, circEnd: int, assembly: str, b
             return combo, gene, transcript
 
 def process_file(file, assembly: str, file_type, new_file, task_id, bias=2):
+    engine.dispose()
+    connection = engine.raw_connection()
 
     cols = {'bed': [0, 1, 2],
             'ciri': [1, 2, 3]}
@@ -168,73 +173,84 @@ def process_file(file, assembly: str, file_type, new_file, task_id, bias=2):
             valid_line = True
             try:
                 sl = line.split()
-                print("Before try", l)
-                l = [sl[i] for i in cols[file_type]]
-                print("L":, l)
+                l = [sl[i] for i in cols[file_type.lower()]]
                 assert l[0].lower().startswith('chr')
                 assert len(l[0][3:-1]) <= 3
                 assert int(l[1]) < int(l[2])
             except:
                 valid_line = False
-
-            print(valid_line)
+            
+            print("Line:", valid_line)
 
             if valid_line:
                 # map circ to annotated_circ
                 chr_num = l[0].lower()
                 start = int(l[1])
                 end = int(l[2])
-
+                time1 = time.time()
                 result_script = f'select * from {assembly}_circRNAs where chr_num="{chr_num}" and start>={start-bias} and start<={start+bias} and end<={end+bias} and end>={end-bias};'
                 with connection.cursor() as cur:
                     cur.execute(result_script)
                     result = cur.fetchall()
+                time2 = time.time()
 
-                print('Query MySQL', len(result))
+                print('Query MySQL1', len(result), f'Used {round(time2-time1,2)}s')
 
                 possible_circ = [
                     (i[4] - i[3] - (end - start), i) for i in result]
 
                 if len(possible_circ) > 0:
                     # append to circ_on_gene
                     circ = sorted(possible_circ, key=lambda x: x[0])[
-                                  0][1].dict()
-                    circ['source'] = 'CIRCpedia V2'
+                                  0][1]
+                    circ_json = {"start": int(circ[3]),
+                            "end": int(circ[4]),
+                            "source": "CIRCpedia V2",
+                            "gene": circ[0],
+                            "transcript": circ[1],
+                            "components": circ[-1]}
 
                     try:
-                        circ_on_gene[circ['gene']][1].append(circ)
+                        circ_on_gene[circ_json['gene']][1].append(circ_json)
                     except:
                          # get gene info
-                        geneINFO_script = f'''select * from {assembly}_genome_genes where gene="{circ['gene']}";'''
-
+                        geneINFO_script = f'''select * from {assembly}_genome_genes where id="{circ_json['gene']}";'''
+                        time3 = time.time()
                         with connection.cursor() as cur:
                             cur.execute(geneINFO_script)
                             geneINFO = cur.fetchall()
+                        time4 = time.time()
+                        print('Query MySQL2', f'Used {round(time4-time3,2)}s')
 
-                        circ_on_gene[geneINFO[0].id] = [
-                            geneINFO[0].dict(), [circ]]
+                        circ_on_gene[geneINFO[0][4]] = [
+                            list(geneINFO[0]), [circ]]
 
                 else:
                     unmap_circ.append(l)
-
+    print('Unmapped circ number:', len(unmap_circ))
     for circ in unmap_circ:
         combo, gene, transcript = find_exon_combo(circ[0].lower(), int(circ[1]), int(circ[2]), assembly)
+        print(gene, transcript)
         if combo is not None:
             try:
                 circ_on_gene[gene][1].append({"start": int(circ[1]),
                                                     "end": int(circ[2]),
                                                     "source": "circDraw_annotated",
+                                                    "gene": gene,
+                                                    "transcript": transcript,
                                                     "components": combo})
             except:
                     # get gene info
-                geneINFO_script = f'select * from {assembly}_genome_genes where gene="{gene}";'
+                geneINFO_script = f'select * from {assembly}_genome_genes where id="{gene}";'
                 with connection.cursor() as cur:
                     cur.execute(geneINFO_script)
                     geneINFO = cur.fetchall()
 
                 circ_on_gene[geneINFO[4]] = [geneINFO, [{"start": int(circ[1]),
                                                     "end": int(circ[2]),
                                                     "source": "circDraw_annotated",
+                                                    "gene": gene,
+                                                    "transcript": transcript,
                                                     "components": combo}]]
 
     print('Mapped circ:', len(circ_on_gene))
@@ -330,6 +346,6 @@ def handle(config):
         return True,circRNA_length_distribution,circRNA_isoform
     except Exception as e:
         print('Handle Error:', e)
-        return False,circRNA_length_distribution,circRNA_isoform
+        return False,{"x":[],"y":[]},{"x":[],"y":[]}
 
 

requirements.txt

@@ -1,18 +1,65 @@
 argh==0.26.2
-Django==2.1.4
+asn1crypto==0.24.0
+attrs==17.4.0
+Automat==0.6.0
+cached-property==1.4.0
+certifi==2018.8.24
+chardet==3.0.4
+click==7.0.dev0
+colorama==0.3.7
+command-not-found==0.3
+configobj==5.0.6
+constantly==15.1.0
+cryptography==2.3
+cx-Oracle==7.2.0
+distro-info===0.18ubuntu0.18.10.2
+Django==2.2.3
 django-annoying==0.10.4
+django-db-connection-pool==1.0.1
+docker==3.4.1
+docker-compose==1.21.0
+docker-pycreds==0.2.2
+dockerpty==0.4.1
+docopt==0.6.2
+httplib2==0.11.3
+hyperlink==17.3.1
+idna==2.6
 ijson==2.3
+incremental==16.10.1
+intervaltree==3.0.2
+jsonschema==2.6.0
+language-selector==0.1
 mysql-connector-python==8.0.13
+netifaces==0.10.4
 numpy==1.15.4
-pathtools==0.1.2
+PAM==0.4.2
 pathtools3==0.2.1
 protobuf==3.6.1
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+PyGObject==3.30.1
 PyMySQL==0.9.3
-pyobjc-core==5.1.2
-pyobjc-framework-Cocoa==5.1.2
-pyobjc-framework-FSEvents==5.1.2
-pytz==2018.7
+pyOpenSSL==18.0.0
+pyserial==3.4
+python-apt==1.7.1
+python-debian==0.1.33
+pytz==2019.1
 PyYAML==5.1
+requests==2.18.4
+requests-unixsocket==0.1.5
+service-identity==16.0.0
 six==1.12.0
-uWSGI==2.0.17.1
+sortedcontainers==2.1.0
+SQLAlchemy==1.3.5
+sqlparse==0.3.0
+ssh-import-id==5.7
+systemd-python==234
+texttable==1.4.0
+Twisted==18.7.0
+ufw==0.36
+ujson==1.35
+unattended-upgrades==0.1
+urllib3==1.22
 watchdog3==0.8.5
+websocket-client==0.53.0
+zope.interface==4.3.2