Skip to content

Commit

Permalink
Fixed up serialization issues around Geoshape. thinkaurelius#1183
Browse files Browse the repository at this point in the history
Added support for type embedded GraphSON and Gryo. Also included tests against Graph of The Gods to ensure the full write/read cycle worked.
  • Loading branch information
spmallette committed Nov 17, 2015
1 parent a1aae99 commit 6dfc816
Show file tree
Hide file tree
Showing 5 changed files with 207 additions and 23 deletions.
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
package com.thinkaurelius.titan.core.attribute;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.jsontype.TypeSerializer;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Doubles;
Expand All @@ -13,7 +18,15 @@
import com.thinkaurelius.titan.diskstorage.ScanBuffer;
import com.thinkaurelius.titan.diskstorage.WriteBuffer;
import com.thinkaurelius.titan.graphdb.database.idhandling.VariableLong;
import com.thinkaurelius.titan.graphdb.relations.RelationIdentifier;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONUtil;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;

import java.io.IOException;
import java.lang.reflect.Array;
Expand All @@ -32,13 +45,39 @@

public class Geoshape {

private static String FIELD_TYPE = "type";
private static String FIELD_COORDINATES = "coordinates";
private static String FIELD_RADIUS = "radius";

private static final SpatialContext CTX = SpatialContext.GEO;

/**
* The Type of a shape: a point, box, circle, or polygon
* The Type of a shape: a point, box, circle, or polygon.
*/
public enum Type {
POINT, BOX, CIRCLE, POLYGON;
POINT("Point"),
BOX("Box"),
CIRCLE("Circle"),
POLYGON("Polygon");

private final String gsonName;

Type(String gsonName) {
this.gsonName = gsonName;
}

public boolean gsonEquals(String otherGson) {
return otherGson != null && gsonName.equals(otherGson);
}

public static Type fromGson(String gsonShape) {
return Type.valueOf(gsonShape.toUpperCase());
}

@Override
public String toString() {
return gsonName;
}
}

//coordinates[0] = latitudes, coordinates[1] = longitudes
Expand Down Expand Up @@ -507,6 +546,39 @@ public void write(WriteBuffer buffer, Geoshape attribute) {
}

/**
* Serializer for TinkerPop's Gryo.
*/
public static class GeoShapeGryoSerializer extends Serializer<Geoshape> {
@Override
public void write(Kryo kryo, Output output, Geoshape geoshape) {
float[][] coordinates = geoshape.coordinates;
assert (coordinates.length==2);
assert (coordinates[0].length==coordinates[1].length && coordinates[0].length>0);
int length = coordinates[0].length;
output.writeLong(length);
for (int i = 0; i < 2; i++) {
for (int j = 0; j < length; j++) {
output.writeFloat(coordinates[i][j]);
}
}
}

@Override
public Geoshape read(Kryo kryo, Input input, Class<Geoshape> aClass) {
long l = input.readLong();
assert l>0 && l<Integer.MAX_VALUE;
int length = (int)l;
float[][] coordinates = new float[2][];
for (int i = 0; i < 2; i++) {
coordinates[i] = input.readFloats(length);
}
return new Geoshape(coordinates);
}
}

/**
* Serialization of Geoshape for JSON purposes uses the standard GeoJSON(http://geojson.org/) format.
*
* @author Bryn Cooke
*/
public static class GeoshapeGsonSerializer extends StdSerializer<Geoshape> {
Expand All @@ -518,15 +590,14 @@ public GeoshapeGsonSerializer() {
@Override
public void serialize(Geoshape value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException {
jgen.writeStartObject();
jgen.writeFieldName("type");

jgen.writeFieldName(FIELD_TYPE);

switch(value.getType()) {
case POLYGON:
throw new UnsupportedOperationException("Ploygons are not supported");
throw new UnsupportedOperationException("Polygons are not supported");
case BOX:
jgen.writeString("Polygon");
jgen.writeFieldName("coordinates");
jgen.writeString(Type.BOX.toString());
jgen.writeFieldName(FIELD_COORDINATES);
jgen.writeStartArray();

jgen.writeStartArray();
Expand All @@ -552,26 +623,50 @@ public void serialize(Geoshape value, JsonGenerator jgen, SerializerProvider pro
jgen.writeEndArray();
break;
case CIRCLE:
jgen.writeString("Circle");
jgen.writeFieldName("radius");
jgen.writeString(Type.CIRCLE.toString());
jgen.writeFieldName(FIELD_RADIUS);
jgen.writeNumber(value.getRadius());
jgen.writeFieldName("coordinates");
jgen.writeFieldName(FIELD_COORDINATES);
jgen.writeStartArray();
jgen.writeNumber(value.coordinates[1][0]);
jgen.writeNumber(value.coordinates[0][0]);
jgen.writeEndArray();
break;
case POINT:
jgen.writeString("Point");
jgen.writeFieldName("coordinates");
jgen.writeString(Type.POINT.toString());
jgen.writeFieldName(FIELD_COORDINATES);
jgen.writeStartArray();
jgen.writeNumber(value.coordinates[1][0]);
jgen.writeNumber(value.coordinates[0][0]);
jgen.writeEndArray();
break;
}
jgen.writeEndObject();
}

@Override
public void serializeWithType(Geoshape geoshape, JsonGenerator jgen, SerializerProvider serializerProvider,
TypeSerializer typeSerializer) throws IOException, JsonProcessingException {
jgen.writeStartObject();
if (typeSerializer != null) jgen.writeStringField(GraphSONTokens.CLASS, Geoshape.class.getName());
GraphSONUtil.writeWithType(FIELD_COORDINATES, geoshape.coordinates, jgen, serializerProvider, typeSerializer);
jgen.writeEndObject();
}
}

public static class GeoshapeGsonDeserializer extends StdDeserializer<Geoshape> {
public GeoshapeGsonDeserializer() {
super(Geoshape.class);
}

@Override
public Geoshape deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
// move the parser forward
jsonParser.nextToken();

float[][] f = jsonParser.readValueAs(float[][].class);
jsonParser.nextToken();
return new Geoshape(f);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.thinkaurelius.titan.graphdb.tinkerpop;

import com.thinkaurelius.titan.core.attribute.Geoshape;
import com.thinkaurelius.titan.graphdb.relations.RelationIdentifier;
import com.thinkaurelius.titan.graphdb.tinkerpop.io.graphson.TitanGraphSONModule;
import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
Expand All @@ -8,6 +9,7 @@

/**
* @author Matthias Broecheler ([email protected])
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
public class TitanIoRegistry extends AbstractIoRegistry {

Expand All @@ -18,5 +20,6 @@ public class TitanIoRegistry extends AbstractIoRegistry {
public TitanIoRegistry() {
register(GraphSONIo.class, null, TitanGraphSONModule.getInstance());
register(GryoIo.class, RelationIdentifier.class, null);
register(GryoIo.class, Geoshape.class, new Geoshape.GeoShapeGryoSerializer());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.jsontype.TypeDeserializer;
import com.fasterxml.jackson.databind.jsontype.TypeSerializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
Expand All @@ -22,13 +22,12 @@
*/
public class TitanGraphSONModule extends SimpleModule {

private static final String FIELD_RELATION_ID = "relationId";

private TitanGraphSONModule() {
addSerializer(RelationIdentifier.class, new RelationIdentifierSerializer());
addSerializer(Geoshape.class, new Geoshape.GeoshapeGsonSerializer());

addDeserializer(RelationIdentifier.class, new RelationIdentifierDeserializer());
addDeserializer(Geoshape.class, new Geoshape.GeoshapeGsonDeserializer());
}

private static final TitanGraphSONModule INSTANCE = new TitanGraphSONModule();
Expand All @@ -52,10 +51,10 @@ public void serialize(final RelationIdentifier relationIdentifier, final JsonGen
@Override
public void serializeWithType(final RelationIdentifier relationIdentifier, final JsonGenerator jsonGenerator,
final SerializerProvider serializerProvider, final TypeSerializer typeSerializer) throws IOException, JsonProcessingException {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField(GraphSONTokens.CLASS, RelationIdentifier.class.getName());
jsonGenerator.writeStringField(FIELD_RELATION_ID, relationIdentifier.toString());
jsonGenerator.writeEndObject();
jsonGenerator.writeStartArray();
jsonGenerator.writeString(RelationIdentifier.class.getName());
jsonGenerator.writeString(relationIdentifier.toString());
jsonGenerator.writeEndArray();
}
}

Expand All @@ -66,10 +65,7 @@ public RelationIdentifierDeserializer() {

@Override
public RelationIdentifier deserialize(final JsonParser jsonParser, final DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
if (!jsonParser.getText().equals(FIELD_RELATION_ID)) throw new IOException(String.format("Invalid serialization format for %s", RelationIdentifier.class));
final RelationIdentifier ri = RelationIdentifier.parse(jsonParser.nextTextValue());
jsonParser.nextToken();
return ri;
return RelationIdentifier.parse(jsonParser.getValueAsString());
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package com.thinkaurelius.titan.graphdb;

import com.thinkaurelius.titan.example.GraphOfTheGodsFactory;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.junit.Test;

import java.io.FileInputStream;
import java.io.FileOutputStream;

/**
* Tests Titan specific serialization classes not covered by the TinkerPop suite.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
public abstract class TitanIoTest extends TitanGraphBaseTest {

@Test
public void testGeoShapeSerializationReadWriteAsGraphSONEmbedded() throws Exception {
GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
GraphSONMapper m = graph.io(IoCore.graphson()).mapper().embedTypes(true).create();
GraphWriter writer = graph.io(IoCore.graphson()).writer().mapper(m).create();
FileOutputStream fos = new FileOutputStream("/tmp/test.json");
writer.writeGraph(fos, graph);

clearGraph(config);
open(config);

GraphReader reader = graph.io(IoCore.graphson()).reader().mapper(m).create();
FileInputStream fis = new FileInputStream("/tmp/test.json");
reader.readGraph(fis, graph);

TitanIndexTest.assertGraphOfTheGods(graph);
}

@Test
public void testGeoShapeSerializationReadWriteAsGryo() throws Exception {
GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
graph.io(IoCore.gryo()).writeGraph("/tmp/test.kryo");

clearGraph(config);
open(config);

graph.io(IoCore.gryo()).readGraph("/tmp/test.kryo");

TitanIndexTest.assertGraphOfTheGods(graph);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package com.thinkaurelius.titan.graphdb.inmemory;

import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.diskstorage.configuration.ModifiableConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.WriteConfiguration;
import com.thinkaurelius.titan.graphdb.TitanGraphBaseTest;
import com.thinkaurelius.titan.graphdb.TitanIoTest;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;

import java.util.Map;

/**
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
public class InMemoryTitanIoTest extends TitanIoTest {
public WriteConfiguration getConfiguration() {
ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration();
config.set(GraphDatabaseConfiguration.STORAGE_BACKEND,"inmemory");
return config.getConfiguration();
}

@Override
public void clopen(Object... settings) {
if (settings!=null && settings.length>0) {
if (graph!=null && graph.isOpen()) {
Preconditions.checkArgument(!graph.vertices().hasNext() &&
!graph.edges().hasNext(),"Graph cannot be re-initialized for InMemory since that would delete all data");
graph.close();
}
Map<TitanGraphBaseTest.TestConfigOption,Object> options = validateConfigOptions(settings);
ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration();
config.set(GraphDatabaseConfiguration.STORAGE_BACKEND,"inmemory");
for (Map.Entry<TitanGraphBaseTest.TestConfigOption,Object> option : options.entrySet()) {
config.set(option.getKey().option, option.getValue(), option.getKey().umbrella);
}
open(config.getConfiguration());
}
newTx();
}
}

0 comments on commit 6dfc816

Please sign in to comment.