mereged julies code

.gitignore

@@ -3,7 +3,7 @@ target/
 *.DS_store
 .idea
 *~
-ols-loader/src/main/resources/data
+data
 ols-loader/src/test/resources/tmp
 ols-solr/src/main/resources/solr-conf/ontology/data
 ols-mongo/src/main/resources/mongodb

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "BioSolr"]
+	path = BioSolr
+	url = [email protected]:simonjupp/BioSolr.git

README.md

@@ -1,14 +1,11 @@
 # OLS
 Ontology Lookup Service from SPOT at EBI. 
 
-ols-core	- Core interfaces for the API
-
-ols-loader	- Spring Boot application for checking for ontology updates and indexing new ontologies
+ols-apps - Spring boot applications for creating ontology indexes
+  - ols-solr-app - Spring boot application for creating a SOLR index from an ontology
 
-ols-mongo	- Spring Boot application for creating a mongo-db ontology index
-
-ols-solr  - Spring Boot application for creating a SOLR ontology index
+ols-core	- Core interfaces for the API
 
-ols-web	 - Spring Boot application for the OLS web services and website
+ols-solr  - SOLR ontology indexing module
 
 ontology-tools - Abstract API for parsing and working with ontologies 

ols-apps/ols-loading-app/pom.xml

@@ -3,13 +3,50 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>ols-apps</artifactId>
-        <groupId>uk.ac.ebi.ols</groupId>
-        <version>3.0-SNAPSHOT</version>
+        <groupId>uk.ac.ebi.spot</groupId>
+        <artifactId>ols-parent</artifactId>
+        <version>3.0.0-SNAPSHOT</version>
+        <relativePath>../../ols-parent/pom.xml</relativePath>
     </parent>
+
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ols-loading-app</artifactId>
 
+    <dependencies>
+
+        <dependency>
+            <groupId>uk.ac.ebi.spot</groupId>
+            <artifactId>ols-solr</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 
 </project>

ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java

@@ -0,0 +1,153 @@
+package uk.ac.ebi.spot.ols;
+
+import org.apache.commons.cli.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import uk.ac.ebi.spot.ols.service.OntologyRepositoryService;
+import uk.ac.ebi.spot.ols.model.OntologyDocument;
+import uk.ac.ebi.spot.ols.service.FileUpdatingService;
+import uk.ac.ebi.spot.ols.service.OntologyIndexingService;
+import uk.ac.ebi.spot.ols.model.Status;
+import uk.ac.ebi.spot.ols.model.OntologyIndexer;
+import uk.ac.ebi.spot.ols.util.FileUpdater;
+
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * @author Simon Jupp
+ * @date 11/02/2015
+ * Samples, Phenotypes and Ontologies Team, EMBL-EBI
+ *
+ * This application reads from an ontology documents repository and checks if any ontologies external
+ * ontologies have been updated. If they have it create indexes
+ *
+ */
+@SpringBootApplication
+public class LoadingApplication implements CommandLineRunner {
+
+    private Logger log = LoggerFactory.getLogger(getClass());
+
+    public Logger getLog() {
+        return log;
+    }
+
+    @Autowired
+    OntologyRepositoryService ontologyRepositoryService;
+
+    @Autowired
+    OntologyIndexingService ontologyIndexingService;
+
+    @Autowired
+    List<OntologyIndexer> indexers;
+
+    @Autowired
+    FileUpdater fileUpdater;
+
+    private static String [] ontologies = {};
+
+    @Override
+    public void run(String... args) throws Exception {
+
+        int parseArgs = parseArguments(args);
+
+
+        System.setProperty("entityExpansionLimit", "10000000");
+        Map<OntologyDocument, Exception> failedOntologies = new HashMap<>();
+
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setCorePoolSize(5);
+        executor.setMaxPoolSize(10);
+        executor.setQueueCapacity(25);
+        executor.initialize();
+
+        List<OntologyDocument> allDocuments = new ArrayList<OntologyDocument>();
+
+        if (ontologies.length > 0) {
+            for (String ontologyName : ontologies) {
+                allDocuments.add(ontologyRepositoryService.get(ontologyName));
+            }
+        }
+        else {
+            allDocuments = ontologyRepositoryService.getAllDocuments();
+        }
+
+        CountDownLatch latch = new CountDownLatch(allDocuments.size());
+        FileUpdatingService service = new FileUpdatingService(ontologyRepositoryService, executor, latch);
+        service.checkForUpdates(allDocuments, fileUpdater, ontologies.length>0);
+
+        // wait for ontologies to have been checked
+        latch.await();
+
+        // For all ontologies set to load, create the new index
+        for (OntologyDocument document : ontologyRepositoryService.getAllDocumentsByStatus(Status.TOLOAD)) {
+            try {
+                ontologyIndexingService.indexOntologyDocument(document);
+            } catch (Exception e) {
+                getLog().error("Failed to create any indexes for " + document.getOntologyId());
+            }
+        }
+
+        System.exit(0);
+    }
+
+
+    private static int parseArguments(String[] args) {
+
+        CommandLineParser parser = new GnuParser();
+        HelpFormatter help = new HelpFormatter();
+        Options options = bindOptions();
+
+        int parseArgs = 0;
+        try {
+            CommandLine cl = parser.parse(options, args, true);
+
+            // check for mode help option
+            if (cl.hasOption("") || cl.hasOption("h")) {
+                // print out mode help
+                help.printHelp("ols-loader.sh", options, true);
+                parseArgs += 1;
+            }
+            else {
+                // find -f option to see if we are to force load
+                if (cl.hasOption("f") ) {
+                    ontologies = cl.getOptionValues("f");
+                }
+            }
+        }
+        catch (ParseException e) {
+            System.err.println("Failed to read supplied arguments");
+            help.printHelp("publish", options, true);
+            parseArgs += 1;
+        }
+        return parseArgs;
+    }
+
+    private static Options bindOptions() {
+        Options options = new Options();
+
+        // help
+        Option helpOption = new Option("h", "help", false, "Print the help");
+        options.addOption(helpOption);
+
+        // add output file arguments
+        Option force = new Option("f", "force", true,
+                "List the ontologies to force update");
+        force.setRequired(false);
+        force.setArgs(Option.UNLIMITED_VALUES);
+        options.addOption(force);
+
+        return options;
+    }
+
+    public static void main(String[] args) throws Exception {
+        SpringApplication.run(LoadingApplication.class, args);
+    }
+
+}

ols-apps/ols-loading-app/src/main/resources/application.properties

@@ -0,0 +1,2 @@
+spring.data.mongodb.database ols
+ols.loader.filedir data

ols-loader/src/test/java/uk/ac/ebi/spot/FileUpdateTests.java → ols-apps/ols-loading-app/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java

@@ -1,12 +1,8 @@
-package uk.ac.ebi.spot;
+package uk.ac.ebi.spot.ols;
 
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.SpringApplicationConfiguration;
 import org.springframework.core.env.Environment;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import uk.ac.ebi.spot.ols.exception.FileUpdateServiceException;
+import uk.ac.ebi.spot.ols.util.FileUpdater;
 
 import java.net.URI;
 
@@ -15,18 +11,18 @@
  * @date 16/02/2015
  * Samples, Phenotypes and Ontologies Team, EMBL-EBI
  */
-@RunWith(SpringJUnit4ClassRunner.class)
-@SpringApplicationConfiguration(classes = FileUpdater.class)
+//@RunWith(SpringJUnit4ClassRunner.class)
+//@SpringApplicationConfiguration(classes = FileUpdater.class)
 public class FileUpdateTests {
 
-    @Autowired
+//    @Autowired
     FileUpdater updater;
 
-    @Autowired
+//    @Autowired
     private Environment environment;
 
 
-    @Test
+//    @Test
     public void testFileUpdater () {
 
         updater.setPath("/Users/jupp/Dropbox/dev/ols/ols-loader/src/test/resources/tmp");

ols-apps/ols-solr-app/README.md

@@ -0,0 +1,61 @@
+# OLS-SOLR
+This module is a spring boot application for creating a SOLR index from a given ontology. 
+
+You need a local version of SOLR running, we are currently testing with version 4.10.3. 
+
+Start SOLR in the example directory with the config supplied by this module 
+
+e.g. 
+
+java -Dsolr.solr.home=<PATH TO HERE>/ols/ols-solr/src/main/solr-conf -jar start.jar
+
+Once the SOLR service is running (by default at http://localhost:8983/solr) you can create a new index as follows:
+
+1. Create an ontology configuration file. There are some examples in src/main/resources/*.properties
+2. In the project root build the OLS application jars using: mvn clean package
+3. Assuming a properties file called application-uberon.properties you can run the application to build the index with: 
+
+java -Xmx2g -jar -Dspring.profiles.active=uberon ols-apps/ols-solr-app/target/ols-solr-app-3.0-SNAPSHOT.jar
+
+The config for application-uberon.properties would include:
+
+```
+# The ontology URI
+ontology_uri  http://purl.obolibrary.org/obo/uberon.owl
+
+# The full name of the ontology
+title  Uber Anatomy Ontology
+
+# The short name for this ontology
+namespace UBERON
+
+# The location to download this ontology (can also be local file path e.g. file:/tmp/uberon.owl
+location http://purl.obolibrary.org/obo/uberon.owl
+
+# primary term label property
+label_property  http://www.w3.org/2000/01/rdf-schema#label
+
+# term definition property (use , for multiple)
+definition_property http://purl.obolibrary.org/obo/IAO_0000115
+
+# term synonym property (use , for multiple)
+synonym_property    http://www.geneontology.org/formats/oboInOwl#hasExactSynonym
+
+# experimental, can ignore for now
+hierarchical_property   http://purl.obolibrary.org/obo/BFO_0000050
+
+# list any properties where you want to ignore assertions (can be annotation or object properties)
+hidden_property
+
+# Base URIs that are local to this ontology, used to identify terms that are defined in this ontology. 
+base_uri    http://purl.obolibrary.org/obo/UBERON_,http://purl.obolibrary.org/obo/UBPROP_,http://purl.obolibrary.org/obo/uberon/core#
+
+# ignore this
+isInferred false
+
+# does this ontology need to be classified with a DL reasoner? default is ELK, if true we will classify with Hermit
+classify false
+
+# True if the ontology contains OBO style slim annotations
+oboSlims true
+```

ols-apps/ols-solr-app/pom.xml

@@ -3,13 +3,38 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>ols-apps</artifactId>
-        <groupId>uk.ac.ebi.ols</groupId>
-        <version>3.0-SNAPSHOT</version>
+        <groupId>uk.ac.ebi.spot</groupId>
+        <artifactId>ols-parent</artifactId>
+        <version>3.0.0-SNAPSHOT</version>
+        <relativePath>../../ols-parent/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ols-solr-app</artifactId>
 
 
+    <dependencies>
+        <dependency>
+            <groupId>uk.ac.ebi.spot</groupId>
+            <artifactId>ols-solr</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+   		<plugins>
+   			<plugin>
+   				<groupId>org.springframework.boot</groupId>
+   				<artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+   			</plugin>
+   		</plugins>
+   	</build>
+
+
 </project>