Skip to content

Commit

Permalink
API / Extent / Add records/uuid/extent.json to list record extents an…
Browse files Browse the repository at this point in the history
…d add extent/0.png for returning only one (geonetwork#4930)

* refactor

* nominal test

* testing for if-modified-since

* refactor

* testing with two extents

* make the tests run

* refactor

* getting third extent polygon

* also returning bouding box alone

* polygons preferred over bounding boxes

* refactor

* seems that gml parser are not thread safe

* minor doc changes (geonetwork#4923)

Co-authored-by: david blasby <[email protected]>

* API / Extent / Allow to request only one geometry

Operation `GET /api/records{metadataUuid}/extents.json` return a list of available extent.

```json
[
{
href: "http://localhost:8080/geonetwork/srv/api/records/9fa0dbe1-1a0d-4fbb-86ad-e7729b9697d4/extents.png",
type: "ALL",
xpath: "",
description: ""
},
{
href: "http://localhost:8080/geonetwork/srv/api/records/9fa0dbe1-1a0d-4fbb-86ad-e7729b9697d4/extents/1.png",
type: "EX_GeographicBoundingBox",
xpath: "/mdb:MD_Metadata/mdb:identificationInfo/mri:MD_DataIdentification/mri:extent[2]/gex:EX_Extent/gex:geographicElement/gex:EX_GeographicBoundingBox",
description: "Planet earth"
},
{
href: "http://localhost:8080/geonetwork/srv/api/records/9fa0dbe1-1a0d-4fbb-86ad-e7729b9697d4/extents/2.png",
type: "EX_GeographicBoundingBox",
xpath: "/mdb:MD_Metadata/mdb:identificationInfo/mri:MD_DataIdentification/mri:extent[3]/gex:EX_Extent/gex:geographicElement[1]/gex:EX_GeographicBoundingBox",
description: ""
},
{
href: "http://localhost:8080/geonetwork/srv/api/records/9fa0dbe1-1a0d-4fbb-86ad-e7729b9697d4/extents/3.png",
type: "EX_BoundingPolygon",
xpath: "/mdb:MD_Metadata/mdb:identificationInfo/mri:MD_DataIdentification/mri:extent[3]/gex:EX_Extent/gex:geographicElement[2]/gex:EX_BoundingPolygon",
description: ""
},
{
href: "http://localhost:8080/geonetwork/srv/api/records/9fa0dbe1-1a0d-4fbb-86ad-e7729b9697d4/extents/4.png",
type: "EX_GeographicBoundingBox",
xpath: "/mdb:MD_Metadata/mdb:identificationInfo/mri:MD_DataIdentification/mri:extent[3]/gex:EX_Extent/gex:geographicElement[3]/gex:EX_GeographicBoundingBox",
description: ""
}
]
```

Operation `GET /api/records{metadataUuid}/extents.png` return all extents combined in one image (same as before).

Operation `GET /api/records{metadataUuid}/extents/{extentIndex}.json` return the requested extent. The extent can be a bounding box or a bounding polygon.

This is only supported for ISO19139 and ISO19115-3 standards.

* API / Extent / Allow to request only one geometry - A bit more robust for schema not having geometry.

* refactor

* refactor, rename test

* to be sure

* Revert "minor doc changes (geonetwork#4923)"

This reverts commit 895ceb5.

* gml parser not thread-safe

* systematically create gml parser when one needed

* refactor

* use map renderer for pdf generation so not to have credential to give

* fix tests

* API / Extent / Formatters / XSL / Display all geometry separately. Preserve order of XML for all geographicElement.

* API / Extent / Formatters / XSL / Display all geometry separately. Preserve order of XML for all geographicElement.

* API / Extent / Fix projection when requesting one polygon.

* API / Extent / Fix test with correct coordinates - added overview image to check file on disk.

* editor: handle invalid srsName for geometry edit panel

* editor: surround gml geom with gmd:polygon on directive load

* update ol - urn:x-ogc:def:crs:EPSG:6.6:4326

Co-authored-by: davidblasby <[email protected]>
Co-authored-by: david blasby <[email protected]>
Co-authored-by: Francois Prunayre <[email protected]>
Co-authored-by: Florent gravin <[email protected]>
  • Loading branch information
5 people authored Sep 28, 2020
1 parent 74f655e commit c5be401
Show file tree
Hide file tree
Showing 32 changed files with 2,173 additions and 250 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ dist: trusty
sudo: false
language: java
jdk:
- oraclejdk8
- openjdk8
before_install:
- unset _JAVA_OPTIONS
- export MAVEN_OPTS="-Xmx512M -XX:MaxPermSize=256M"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.geotools.xsd.Configuration;
import org.geotools.xsd.Parser;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.index.SpatialIndex;
Expand Down Expand Up @@ -90,15 +92,12 @@
import org.fao.geonet.kernel.search.spatial.TouchesFilter;
import org.fao.geonet.kernel.search.spatial.WithinBBoxFilter;
import org.fao.geonet.kernel.setting.SettingInfo;
import org.fao.geonet.util.GMLParsers;
import org.fao.geonet.utils.Xml;
import org.geotools.data.DataStore;
import org.geotools.data.DefaultTransaction;
import org.geotools.data.FeatureSource;
import org.geotools.data.Transaction;
import org.geotools.gml3.GMLConfiguration;
import org.geotools.xsd.Configuration;
import org.geotools.xsd.Parser;

import org.jdom.Content;
import org.jdom.Element;
import org.opengis.feature.simple.SimpleFeature;
Expand Down Expand Up @@ -1550,7 +1549,6 @@ public class Spatial {
private final Map<String, Constructor<? extends SpatialFilter>> _types;
private final Transaction _transaction;
private final int _maxWritesInTransaction;
private final Parser[] _gmlParsers;
private final Lock _lock;
private SpatialIndexWriter _writer;
private volatile Committer _committerTask;
Expand All @@ -1574,7 +1572,6 @@ public class Spatial {
throw new RuntimeException("Unable to create types mapping", e);
}
_types = Collections.unmodifiableMap(types);
_gmlParsers = GMLParsers.create();
}

/**
Expand Down Expand Up @@ -1609,7 +1606,7 @@ public Spatial(DataStore dataStore, int maxWritesInTransaction) throws Exception
private boolean createWriter(DataStore datastore) throws IOException {
boolean rebuildIndex;
try {
_writer = new SpatialIndexWriter(datastore, _gmlParsers, _transaction, _maxWritesInTransaction, _lock);
_writer = new SpatialIndexWriter(datastore, _transaction, _maxWritesInTransaction, _lock);
rebuildIndex = _writer.getFeatureSource().getSchema() == null;
} catch (Throwable e) {

Expand Down Expand Up @@ -1705,7 +1702,7 @@ public SpatialIndexWriter writer() throws Exception {

private SpatialIndexWriter writerNoLocking() throws Exception {
if (_writer == null) {
_writer = new SpatialIndexWriter(_datastore, _gmlParsers, _transaction, _maxWritesInTransaction, _lock);
_writer = new SpatialIndexWriter(_datastore, _transaction, _maxWritesInTransaction, _lock);
}
return _writer;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.locationtech.jts.index.strtree.STRtree;
import org.apache.jcs.access.exception.CacheException;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.util.GMLParsers;
import org.fao.geonet.utils.Log;
import org.fao.geonet.utils.Xml;
import org.geotools.data.DataStore;
Expand Down Expand Up @@ -89,7 +90,6 @@ public class SpatialIndexWriter implements FeatureListener {
public static final int MAX_WRITES_IN_TRANSACTION = 1000;
static final String SPATIAL_FILTER_JCS = "SpatialFilterCache";
private static int _writes;
private final Parser[] _parsers;
private final Transaction _transaction;
private final Lock _lock;
private int _maxWrites;
Expand All @@ -105,17 +105,11 @@ public class SpatialIndexWriter implements FeatureListener {
* @param maxWrites Maximum number of writes in a transaction. If set to 1 then AUTO_COMMIT is
* being used.
*/
public SpatialIndexWriter(DataStore datastore, Parser[] parsers,
Transaction transaction, int maxWrites, Lock lock)
public SpatialIndexWriter(DataStore datastore, Transaction transaction, int maxWrites, Lock lock)
throws Exception {
// Note: The Configuration takes a long time to create so it is worth
// re-using the same Configuration
_lock = lock;
_parsers = parsers;
for (Parser _parser : _parsers) {
_parser.setStrict(false);
_parser.setValidating(false);
}
_transaction = transaction;
_maxWrites = maxWrites;

Expand All @@ -131,13 +125,12 @@ public SpatialIndexWriter(DataStore datastore, Parser[] parsers,
/**
* Extracts a Geometry Collection from metadata default visibility for testing access.
*/
static MultiPolygon extractGeometriesFrom(Path schemaDir,
Element metadata, Parser[] parsers, Map<String, String> errorMessage) throws Exception {
return getSpatialExtent(schemaDir, metadata, parsers, new SpatialIndexingErrorHandler(errorMessage));
static MultiPolygon extractGeometriesFrom(Path schemaDir, Element metadata, Map<String, String> errorMessage) throws Exception {
return getSpatialExtent(schemaDir, metadata, new SpatialIndexingErrorHandler(errorMessage));
}

public static MultiPolygon getSpatialExtent(Path schemaDir, Element metadata, Parser[] parsers, ErrorHandler errorHandler) throws Exception {
org.geotools.util.logging.Logging.getLogger("org.geotools.xsd")
public static MultiPolygon getSpatialExtent(Path schemaDir, Element metadata, ErrorHandler errorHandler) throws Exception {
org.geotools.util.logging.Logging.getLogger("org.geotools.xsd")
.setLevel(Level.SEVERE);
Path sSheet = schemaDir.resolve("extract-gml.xsl").toAbsolutePath();
Element transform = Xml.transform(metadata, sSheet);
Expand All @@ -146,12 +139,7 @@ public static MultiPolygon getSpatialExtent(Path schemaDir, Element metadata, Pa
}
List<Polygon> allPolygons = new ArrayList<Polygon>();
for (Element geom : (List<Element>) transform.getChildren()) {
Parser parser = null;
if (geom.getNamespace().equals(Geonet.Namespaces.GML32)) {
parser = parsers[1]; // geotools gml3.2 parser
} else {
parser = parsers[0];
}
Parser parser = GMLParsers.create(geom);
String srs = geom.getAttributeValue("srsName");
CoordinateReferenceSystem sourceCRS = DefaultGeographicCRS.WGS84;
String gml = Xml.getString(geom);
Expand Down Expand Up @@ -218,7 +206,7 @@ public static MultiPolygon parseGml(Parser parser, String gml) throws IOExceptio
if (value instanceof HashMap) {
@SuppressWarnings("rawtypes")
HashMap map = (HashMap) value;
List<Polygon> geoms = new ArrayList<Polygon>();
List<MultiPolygon> geoms = new ArrayList<MultiPolygon>();
for (Object entry : map.values()) {
addToList(geoms, entry);
}
Expand All @@ -238,14 +226,16 @@ public static MultiPolygon parseGml(Parser parser, String gml) throws IOExceptio
}
}

public static void addToList(List<Polygon> geoms, Object entry) {
public static void addToList(List<MultiPolygon> geoms, Object entry) {
if (entry instanceof Polygon) {
geoms.add((Polygon) entry);
geoms.add(toMultiPolygon((Polygon) entry));
} else if (entry instanceof MultiPolygon) {
geoms.add((MultiPolygon) entry);
} else if (entry instanceof Collection) {
@SuppressWarnings("rawtypes")
Collection collection = (Collection) entry;
for (Object object : collection) {
geoms.add((Polygon) object);
geoms.add(toMultiPolygon((Polygon) object));
}
}
}
Expand All @@ -267,7 +257,7 @@ public void index(Path schemaDir, String id,
_index = null;
errorMessage = new HashMap<>();
Geometry geometry = extractGeometriesFrom(
schemaDir, metadata, _parsers, errorMessage);
schemaDir, metadata, errorMessage);

if (geometry != null && !geometry.getEnvelopeInternal().isNull()) {
MemoryFeatureCollection features = new MemoryFeatureCollection(_featureStore.getSchema());
Expand Down
38 changes: 29 additions & 9 deletions core/src/main/java/org/fao/geonet/util/GMLParsers.java
Original file line number Diff line number Diff line change
@@ -1,16 +1,36 @@
package org.fao.geonet.util;

import org.fao.geonet.constants.Geonet;
import org.geotools.gml3.GMLConfiguration;
import org.geotools.xsd.Configuration;
import org.geotools.xsd.Parser;
import org.jdom.Element;

public final class GMLParsers {
public static Parser[] create() {
Parser[] parsers = {
new Parser(new GMLConfiguration()),
new Parser(new org.geotools.gml3.v3_2.GMLConfiguration())
};
return parsers;
}

private GMLParsers() {}

private static final GMLConfiguration GML_CONFIGURATION = new GMLConfiguration();
private static final org.geotools.gml3.v3_2.GMLConfiguration GML_32_CONFIGURATION = new org.geotools.gml3.v3_2.GMLConfiguration();

static public Parser create(Element geom) {
if (geom.getNamespace().equals(Geonet.Namespaces.GML32)) {
return createGML32();
} else {
return createGML();
}
}

static private Parser createGML32() {
return createParser(GML_32_CONFIGURATION);
}

public static Parser createGML() {
return createParser(GML_CONFIGURATION);
}

static private Parser createParser(Configuration configuration) {
Parser parser = new Parser(configuration);
parser.setStrict(false);
parser.setValidating(false);
return parser;
}
}
8 changes: 1 addition & 7 deletions core/src/main/java/org/fao/geonet/util/XslUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -738,13 +738,7 @@ public static String geomToBbox(Object geom) {
String srs = geomElement.getAttributeValue("srsName");
CoordinateReferenceSystem geomSrs = DefaultGeographicCRS.WGS84;
if (srs != null && !(srs.equals(""))) geomSrs = CRS.decode(srs);
Parser[] parsers = GMLParsers.create();
Parser parser = null;
if (geomElement.getNamespace().equals(Geonet.Namespaces.GML32)) {
parser = parsers[1];
} else {
parser = parsers[0];
}
Parser parser = GMLParsers.create(geomElement);
MultiPolygon jts = parseGml(parser, gml);


Expand Down
78 changes: 78 additions & 0 deletions core/src/test/java/org/fao/geonet/util/GMLParserLearnTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package org.fao.geonet.util;

import org.fao.geonet.utils.Xml;
import org.geotools.xsd.Parser;
import org.jdom.JDOMException;
import org.junit.Test;
import org.xml.sax.SAXException;

import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.io.StringReader;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GCO;
import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GMD;
import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GML;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;

public class GMLParserLearnTest {

static final String TO_PARSE;

static {
URL resource = GMLParserLearnTest.class.getResource("../kernel/multilingual-metadata.xml");
String toParse;
try {
toParse = Xml.getString(Xml.selectElement( Xml.loadStream(resource.openStream()), ".//gmd:polygon", Arrays.asList(GMD, GCO, GML)));
} catch (Exception e) {
toParse = null;
e.printStackTrace();
}
TO_PARSE = toParse;
}


@Test
public void geotoolsGmlParserNotThreadSafe() throws IOException, JDOMException, ParserConfigurationException, SAXException, InterruptedException {
final Parser parser = GMLParsers.createGML();
parser.parse(new StringReader(TO_PARSE));

IntStream range = IntStream.rangeClosed(1, 20);
List<Object> failedParseNew = range.parallel().mapToObj(GMLParserLearnTest::parseCreateNewParser).filter(x -> x == null).collect(Collectors.toList());
assertEquals(0, failedParseNew.size());

range = IntStream.rangeClosed(1, 20);
List<Object> failedParseReuse = range.parallel().mapToObj(GMLParserLearnTest::parseResuseParser).filter(x -> x == null).collect(Collectors.toList());
assertNotEquals(0, failedParseReuse.size());

}

static private Object parseCreateNewParser(int inc) {
Parser parser = GMLParsers.createGML();
try {
return parser.parse(new StringReader(TO_PARSE));
} catch (Exception e) {
//e.printStackTrace();
return null;
}

}

static private Parser parserToReuse = GMLParsers.createGML();

static private Object parseResuseParser(int inc) {
try {
return parserToReuse.parse(new StringReader(TO_PARSE));
} catch (Exception e) {
//e.printStackTrace();
return null;
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -548,34 +548,32 @@
</xsl:template>


<xsl:template mode="render-field"
match="gex:EX_BoundingPolygon/gex:polygon"
priority="100">
<xsl:copy-of select="gn-fn-render:extent($metadataUuid,
count(ancestor::mri:extent/preceding-sibling::mri:extent/*/*[local-name() = 'geographicElement']/*) +
count(../../preceding-sibling::gex:geographicElement) + 1)"/>
<br/>
<br/>
</xsl:template>

<!-- Display spatial extents containing bounding polygons on a map -->

<xsl:template mode="render-field"
match="gex:EX_Extent[gex:geographicElement/*/gex:polygon]"
match="gex:EX_Extent"
priority="100">
<div class="entry name">
<h4>
<xsl:call-template name="render-field-label">
<h4>
<xsl:call-template name="render-field-label">
<xsl:with-param name="languages" select="$allLanguages"/>
</xsl:call-template>
<xsl:apply-templates mode="render-value"
select="@*"/>
</h4>
<div class="target">

<xsl:apply-templates mode="render-field" select="gex:description"/>

<!-- Display all included bounding polygons/boxes on the one map -->

<xsl:copy-of select="gn-fn-render:extent($metadataUuid)"/>

<!-- Display any included geographic descriptions separately after map -->

<xsl:apply-templates mode="render-field" select="gex:geographicElement[gex:EX_GeographicDescription]"/>

<xsl:apply-templates mode="render-field" select="gex:temporalElement"/>
<xsl:apply-templates mode="render-field" select="gex:verticalElement"/>

<xsl:apply-templates mode="render-value"
select="@*"/>
</h4>
<div class="target">
<xsl:apply-templates mode="render-field"
select="gex:*"/>
</div>
</div>
</xsl:template>
Expand Down
Loading

0 comments on commit c5be401

Please sign in to comment.