added README

mSorok · Mar 3, 2020 · 44647ad · 44647ad
1 parent ae735a1
commit 44647ad
Show file tree

Hide file tree

Showing 16 changed files with 232 additions and 52 deletions.
diff --git a/README.md b/README.md
@@ -0,0 +1,86 @@
+# Compiler for the COlleCtion of Open NatUral producTs (COCONUT)
+
+This compiles is designed to read molecules from various file types (SMILES, SDF, MOL, csv, tsv), check them for errors and connectivity, compute a large number of molecular parameters and properties, and store everything in a Mongo database.
+
+#### System pre-requisites:
+
+- MongoDB installed and accessible by the default (27017) port on localhost
+- Java minimum 8 version installed
+
+
+#### Load COCONUT
+You can download the latest version COCONUT from ZENODO (https://zenodo.org/record/3688734). In case you want to explore the whole database in MongoDB, you can load the downloaded dataset dump as following:
+````bash
+unzip COCONUTv2.zip
+cd COCONUTv1/COCONUT/
+mongorestore --db=COCONUT --noIndexRestore .
+````
+
+Note that for system compatibilities reason, it is better to restore the database without the indexes (hence the "noIndexRestore" option).
+However, seen the size of the dataset, we suggest to add the indexes as following:
+
+````bash
+mongo
+use COCONUT
+db.sourceNaturalProduct.createIndex( {source:1})
+db.sourceNaturalProduct.createIndex( {simpleInchi:1})
+db.sourceNaturalProduct.createIndex( {simpleInchiKey:1})
+db.uniqueNaturalProduct.createIndex( {inchi:1})
+db.uniqueNaturalProduct.createIndex( {inchikey:1})
+db.uniqueNaturalProduct.createIndex( {smiles:1})
+db.uniqueNaturalProduct.createIndex( {clean_smiles:1})
+db.uniqueNaturalProduct.createIndex( {molecular_formula:1})
+db.uniqueNaturalProduct.createIndex( {name:1})
+db.uniqueNaturalProduct.createIndex( {coconut_id:1})
+db.fragment.createIndex({signature:1})
+db.fragment.createIndex({signature:1, withsugar:-1})
+````
+
+#### Required folder structure
+
+````bash
+COCONUT
+├── coconut-0.0.1-SNAPSHOT.jar # the compiled jar. We provide is as apart of the curent repository
+├── data # here go the files with NP molecular structures
+├── fragments
+│   ├── fragment_without_sugar.txt
+│   ├── fragment_with_sugar.txt
+├── sm # file(s) with structures of synthetic molecules 
+````
+
+#### Execution options
+
+##### Run COCONUT compilation from scratch
+
+```bash
+java -jar coconut-0.0.1-SNAPSHOT.jar ~/Projects/NP/COCONUT/data ~/Projects/NP/COCONUT/sm/sm.tsv ~/Projects/NP/COCONUT/fragments/fragment_without_sugar.txt ~/Projects/NP/COCONUT/fragments/fragment_with_sugar.txt &
+```
+
+##### Re-run COCONUT to recompute missing molecular features 
+
+```bash
+java -jar coconut-0.0.1-SNAPSHOT.jar recomputeMissing &
+```
+
+##### Run only similarity calculation
+The provided script also computes similatiry between molecules in COCONUT. This step requires a large amount of memory (this will be optimized in the future) and might need a separate calculation.
+````bash
+java -Xmx12288m -jar coconut-0.0.1-SNAPSHOT.jar runOnlySimilarity &
+````
+
+##### Run only addition of synthetic molecules
+Synthetic molecules (SM) are required for a large number of comparisons with NPs (for example a re-calculation from scratch of NP-likeness score). You need to provide your own dataset of synthetic molecules (SM), we suggest using the ZINC15 dataset.
+Note that not adding any SM will not affect COCONUT, unless there is a need of re-calculating the NP-likeness score from scratch.
+
+```bash
+java -jar coconut-0.0.1-SNAPSHOT.jar onlyAddSM ~/Projects/NP/COCONUT/sm/sm.tsv & 
+```
+
+##### Run only CNPid calculation
+In case you need to produce your own CNPid (COCONUT NP identifiers) - not recommened.
+
+```bash
+java -jar coconut-0.0.1-SNAPSHOT.jar addCNPid  & 
+```
+
+
diff --git a/src/main/java/de/unijena/cheminf/npopensourcecollector/NpOpenSourceCollectorApplication.java b/src/main/java/de/unijena/cheminf/npopensourcecollector/NpOpenSourceCollectorApplication.java
@@ -47,6 +47,8 @@ public class NpOpenSourceCollectorApplication implements CommandLineRunner {
     @Autowired
     CreateCNPidService createCNPidService;
 
+
+
     public static void main(String[] args) {
         SpringApplication.run(NpOpenSourceCollectorApplication.class, args);
     }
@@ -126,10 +128,7 @@ else if(args[0].equals("onlyAddSM")){
 
                     fragmentCalculatorService.doParallelizedWork(40);
 
-                    while (!fragmentCalculatorService.processFinished()) {
-                        System.out.println("I'm waiting");
-                        TimeUnit.MINUTES.sleep(1);
-                    }
+
 
                     System.out.println("Done fragmenting natural products");
                     SimpleDateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");

diff --git a/.../java/de/unijena/cheminf/npopensourcecollector/mongocollections/SourceNaturalProduct.java b/.../java/de/unijena/cheminf/npopensourcecollector/mongocollections/SourceNaturalProduct.java
@@ -20,6 +20,8 @@ public class SourceNaturalProduct {
 
     public String simpleSmiles;
 
+    public String absoluteSmiles;
+
     public String originalInchi;
 
     public String originalInchiKey;
@@ -237,4 +239,12 @@ public ArrayList<String> getGeographicLocation() {
     public void setGeographicLocation(ArrayList<String> geographicLocation) {
         this.geographicLocation = geographicLocation;
     }
+
+    public String getAbsoluteSmiles() {
+        return absoluteSmiles;
+    }
+
+    public void setAbsoluteSmiles(String absoluteSmiles) {
+        this.absoluteSmiles = absoluteSmiles;
+    }
 }
diff --git a/.../java/de/unijena/cheminf/npopensourcecollector/mongocollections/UniqueNaturalProduct.java b/.../java/de/unijena/cheminf/npopensourcecollector/mongocollections/UniqueNaturalProduct.java
@@ -41,6 +41,9 @@ public class UniqueNaturalProduct {
 
     public String sugar_free_smiles;
 
+
+    public Hashtable<String, ArrayList<String>> absolute_smiles;
+
     public Integer contains_ring_sugars;
 
     public Integer contains_linear_sugars;
@@ -847,4 +850,12 @@ public String getSubstructureFingerprint() {
     public void setSubstructureFingerprint(String substructureFingerprint) {
         this.substructureFingerprint = substructureFingerprint;
     }
+
+    public Hashtable<String, ArrayList<String>> getAbsolute_smiles() {
+        return absolute_smiles;
+    }
+
+    public void setAbsolute_smiles(Hashtable<String, ArrayList<String>> absolute_smiles) {
+        this.absolute_smiles = absolute_smiles;
+    }
 }
diff --git a/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/CSVReader.java b/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/CSVReader.java
@@ -52,6 +52,7 @@ public CSVReader(){
     public void readFile(File file) {
 
         SmilesGenerator smilesGenerator = new SmilesGenerator(SmiFlavor.Unique );
+        SmilesGenerator absoluteSmilesGenerator = new SmilesGenerator(SmiFlavor.Absolute );
         SmilesParser sp = new SmilesParser(DefaultChemObjectBuilder.getInstance());
         this.file = file;
         this.source = file.getName().toLowerCase().replace(".csv", "");
@@ -227,7 +228,12 @@ else if(indexOfName != null){
 
                             }
 
-                            molecule.setProperty("SIMPLE_SMILES", smilesGenerator.create(molecule));
+                            String simpleSmiles = smilesGenerator.create(molecule);
+                            String absoluteSmiles = absoluteSmilesGenerator.create(molecule);
+                            molecule.setProperty("SIMPLE_SMILES", simpleSmiles);
+                            if(!absoluteSmiles.equals(simpleSmiles)) {
+                                molecule.setProperty("ABSOLUTE_SMILES", absoluteSmiles);
+                            }
 
 
                             DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd");

diff --git a/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/InChiReader.java b/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/InChiReader.java
@@ -54,6 +54,7 @@ public InChiReader(){
     public void readFile(File file) {
 
         SmilesGenerator smilesGenerator = new SmilesGenerator(SmiFlavor.Unique );
+        SmilesGenerator absoluteSmilesGenerator = new SmilesGenerator(SmiFlavor.Absolute );
 
 
         this.file = file;
@@ -147,7 +148,12 @@ public void readFile(File file) {
 
                             }
 
-                            molecule.setProperty("SIMPLE_SMILES", smilesGenerator.create(molecule));
+                            String simpleSmiles = smilesGenerator.create(molecule);
+                            String absoluteSmiles = absoluteSmilesGenerator.create(molecule);
+                            molecule.setProperty("SIMPLE_SMILES", simpleSmiles);
+                            if(!absoluteSmiles.equals(simpleSmiles)) {
+                                molecule.setProperty("ABSOLUTE_SMILES", absoluteSmiles);
+                            }
 
 
                             DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd");

diff --git a/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/MOLReader.java b/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/MOLReader.java
@@ -58,6 +58,8 @@ public MOLReader(){
     public void readFile(File file) {
 
         SmilesGenerator smilesGenerator = new SmilesGenerator(SmiFlavor.Unique); //Unique - canonical SMILES string, different atom ordering produces the same* SMILES. No isotope or stereochemistry encoded.
+        SmilesGenerator absoluteSmilesGenerator = new SmilesGenerator(SmiFlavor.Absolute );
+
 
         this.file = file;
         int count = 1;
@@ -164,7 +166,12 @@ public void readFile(File file) {
                         }
 
 
-                        molecule.setProperty("SIMPLE_SMILES", smilesGenerator.create(molecule));
+                        String simpleSmiles = smilesGenerator.create(molecule);
+                        String absoluteSmiles = absoluteSmilesGenerator.create(molecule);
+                        molecule.setProperty("SIMPLE_SMILES", simpleSmiles);
+                        if(!absoluteSmiles.equals(simpleSmiles)) {
+                            molecule.setProperty("ABSOLUTE_SMILES", absoluteSmiles);
+                        }
 
 
                         DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd");

diff --git a/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/SDFReader.java b/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/SDFReader.java
@@ -65,6 +65,8 @@ public SDFReader(){
     public void readFile(File file) {
 
         SmilesGenerator smilesGenerator = new SmilesGenerator(SmiFlavor.Unique); //Unique - canonical SMILES string, different atom ordering produces the same* SMILES. No isotope or stereochemistry encoded.
+        SmilesGenerator absoluteSmilesGenerator = new SmilesGenerator(SmiFlavor.Absolute );
+
 
         this.file = file;
         int count = 1;
@@ -133,9 +135,13 @@ public void readFile(File file) {
 
                     molecule = tmpMolecule;
 
+                    try {
 
-                    if(!foundOriginalSmiles) {
-                        molecule.setProperty("ORIGINAL_SMILES", smilesGenerator.create(molecule));
+                        if (!foundOriginalSmiles) {
+                            molecule.setProperty("ORIGINAL_SMILES", smilesGenerator.create(molecule));
+                        }
+                    }catch (CDKException e){
+                        e.printStackTrace();
                     }
 
 
@@ -182,7 +188,12 @@ public void readFile(File file) {
                         }
 
 
-                        molecule.setProperty("SIMPLE_SMILES", smilesGenerator.create(molecule));
+                        String simpleSmiles = smilesGenerator.create(molecule);
+                        String absoluteSmiles = absoluteSmilesGenerator.create(molecule);
+                        molecule.setProperty("SIMPLE_SMILES", simpleSmiles);
+                        if(!absoluteSmiles.equals(simpleSmiles)) {
+                            molecule.setProperty("ABSOLUTE_SMILES", absoluteSmiles);
+                        }
 
 
                         DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd");

diff --git a/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/SMILESReader.java b/src/main/java/de/unijena/cheminf/npopensourcecollector/readers/SMILESReader.java
@@ -53,6 +53,7 @@ public SMILESReader(){
     @Override
     public void readFile(File file) {
         SmilesGenerator smilesGenerator = new SmilesGenerator(SmiFlavor.Unique );
+        SmilesGenerator absoluteSmilesGenerator = new SmilesGenerator(SmiFlavor.Absolute );
 
 
         this.file = file;
@@ -142,7 +143,12 @@ public void readFile(File file) {
 
                                 }
 
-                                molecule.setProperty("SIMPLE_SMILES", smilesGenerator.create(molecule));
+                                String simpleSmiles = smilesGenerator.create(molecule);
+                                String absoluteSmiles = absoluteSmilesGenerator.create(molecule);
+                                molecule.setProperty("SIMPLE_SMILES", simpleSmiles);
+                                if(!absoluteSmiles.equals(simpleSmiles)) {
+                                    molecule.setProperty("ABSOLUTE_SMILES", absoluteSmiles);
+                                }
 
 
                                 DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd");
@@ -189,14 +195,14 @@ else if(source.equals("npatlas")){
                             }
 
                         } catch (InvalidSmilesException e) {
-                            e.printStackTrace();
-                            System.out.println(line);
-                            System.out.println(splitted);
-                            System.out.println(splitted[0]);
+                            //e.printStackTrace();
+                            //System.out.println(line);
+                            //System.out.println(splitted);
+                            //System.out.println(splitted[0]);
                             smilesReader.skip(count - 1);
                         }
 
-                    } catch (Exception e) {
+                    } catch (Exception  e) {
                         e.printStackTrace();
                     }
                     count++;

diff --git a/...na/cheminf/npopensourcecollector/services/AtomContainerToSourceNaturalProductService.java b/...na/cheminf/npopensourcecollector/services/AtomContainerToSourceNaturalProductService.java
@@ -28,6 +28,10 @@ public SourceNaturalProduct createSNPlInstance(IAtomContainer ac) {
         np.setSimpleInchiKey(ac.getProperty("SIMPLE_INCHIKEY"));
         np.setSimpleSmiles(ac.getProperty("SIMPLE_SMILES"));
 
+        if(ac.getProperties().containsKey("ABSOLUTE_SMILES")) {
+            np.setAbsoluteSmiles(ac.getProperty("ABSOLUTE_SMILES"));
+        }
+
         np.setTotalAtomNumber(ac.getAtomCount());
 
         int heavyAtomCount = 0;

diff --git a/...ain/java/de/unijena/cheminf/npopensourcecollector/services/FragmentCalculatorService.java b/...ain/java/de/unijena/cheminf/npopensourcecollector/services/FragmentCalculatorService.java
@@ -15,6 +15,7 @@
 import org.openscience.cdk.isomorphism.UniversalIsomorphismTester;
 import org.openscience.cdk.signature.AtomSignature;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
 import java.text.SimpleDateFormat;
@@ -58,12 +59,14 @@ public class FragmentCalculatorService {
     public void doParallelizedWork(int nbThreads){
         System.out.println("Start parallel fragmentation of natural products");
 
+        sugarRemovalService.getSugarPatterns();
+
         try{
 
             List<UniqueNaturalProduct> allNP = uniqueNaturalProductRepository.findAll();
             System.out.println(allNP.size());
 
-            ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(nbThreads);
+            ExecutorService taskExecutor = Executors.newFixedThreadPool(nbThreads);
 
 
             List<List<UniqueNaturalProduct>>  nUniqueMoleculesBatch =  Lists.partition(allNP, 1000);
@@ -82,32 +85,24 @@ public void doParallelizedWork(int nbThreads){
 
                 task.taskid=taskcount;
 
-                Future<?> f = executor.submit(task);
-
-                futures.add(f);
-
-                //executor.execute(task);
-
+                taskExecutor.execute(task);
                 System.out.println("Task "+taskcount+" executing");
 
             }
-
-
-            executor.shutdown();
-            //executor.awaitTermination(210, TimeUnit.SECONDS);
-
-
-
-
-
+            taskExecutor.shutdown();
+            try {
+                taskExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
         } catch (Exception e) {
             e.printStackTrace();
         }
 
 
     }
 
-
+/*
     public boolean processFinished(){
 
         boolean allFuturesDone = true;
@@ -121,12 +116,14 @@ public boolean processFinished(){
         return allFuturesDone;
     }
 
-
+*/
 
     public void doWorkRecompute(){
 
         System.out.println("Start fragmenting natural products for uncomputed");
 
+        sugarRemovalService.getSugarPatterns();
+
 
         List<UniqueNaturalProduct> allNP = uniqueNaturalProductRepository.findAllByNPLScoreComputed();